Improvements to field generation (#137)

* Remove `no-embedded-types` flag

We don't use this.

* Use `fw-download.ubnt.com`

* Return firmware download URL from `latestUnifiVersion`

* Don't use version metadata

fields/api.go.tmpl

@@ -17,11 +17,6 @@
 		{{- else }}
 		dst.{{ .FieldName }} = {{ .FieldType }}(aux.{{ .FieldName }})
 		{{- end }}{{- end }}{{- end }}
-{{ define "field-embed" }}
-	{{ .FieldName }} {{ if .IsArray }}[]{{end}}{{ if not .Fields }}{{ .FieldType }}{{ else }}struct {
-	{{ range $fk, $fv := .Fields }}{{ if not $fv }}
-	{{ else }}{{- template "field-embed" $fv }}{{ end }}{{ end }}
-}{{ end }} `json:"{{ .JSONName }}{{ if .OmitEmpty }},omitempty{{ end }}"` {{ if .FieldValidation }}// {{ .FieldValidation }}{{ end }} {{- end }}
 // Code generated from ace.jar fields *.json files
 // DO NOT EDIT.
 
@@ -40,14 +35,6 @@ var (
 	_ json.Marshaler
 )
 
-{{ if embedTypes -}}
-{{- $k := .StructName -}}
-{{- $v :=index .Types .StructName -}}
-type {{ $k }} struct {
-	{{ range $fk, $fv := $v.Fields }}{{ if not $fv }}
-	{{ else }}{{- template "field-embed" $fv }}{{ end }}{{ end }}
-}
-{{- else -}}
 {{ range $k, $v := .Types }}
 type {{ $k }} struct {
 	{{ range $fk, $fv := $v.Fields }}{{ if not $fv }}
@@ -76,7 +63,6 @@ func (dst *{{ $k }}) UnmarshalJSON(b []byte) error {
 	return nil
 }
 {{ end }}
-{{- end -}}
 
 {{ if not .IsSetting }}
 func (c *Client) list{{ .StructName }}(ctx context.Context, site string) ([]{{ .StructName }}, error) {

fields/extract.go

@@ -8,21 +8,19 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/hashicorp/go-version"
 	"github.com/iancoleman/strcase"
 	"github.com/ulikunitz/xz"
 	"github.com/xor-gate/ar"
 )
 
-func downloadJar(version *version.Version, outputDir string) (string, error) {
+func downloadJar(url *url.URL, outputDir string) (string, error) {
-	url := fmt.Sprintf("https://dl.ui.com/unifi/%s/unifi_sysvinit_all.deb", version)
+	debResp, err := http.Get(url.String())
-
-	debResp, err := http.Get(url)
 	if err != nil {
 		return "", fmt.Errorf("unable to download deb: %w", err)
 	}

fields/fwupdate.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"github.com/hashicorp/go-version"
+)
+
+var firmwareUpdateApi = "https://fw-update.ubnt.com/api/firmware-latest"
+
+const (
+	debianPlatform         = "debian"
+	releaseChannel         = "release"
+	unifiControllerProduct = "unifi-controller"
+)
+
+type firmwareUpdateApiResponse struct {
+	Embedded firmwareUpdateApiResponseEmbedded `json:"_embedded"`
+}
+
+type firmwareUpdateApiResponseEmbedded struct {
+	Firmware []firmwareUpdateApiResponseEmbeddedFirmware `json:"firmware"`
+}
+
+type firmwareUpdateApiResponseEmbeddedFirmware struct {
+	Channel  string                                         `json:"channel"`
+	Created  string                                         `json:"created"`
+	Id       string                                         `json:"id"`
+	Platform string                                         `json:"platform"`
+	Product  string                                         `json:"product"`
+	Version  *version.Version                               `json:"version"`
+	Links    firmwareUpdateApiResponseEmbeddedFirmwareLinks `json:"_links"`
+}
+
+type firmwareUpdateApiResponseEmbeddedFirmwareDataLink struct {
+	Href *url.URL `json:"href"`
+}
+
+func (l firmwareUpdateApiResponseEmbeddedFirmwareDataLink) MarshalJSON() ([]byte, error) {
+	var href string
+	if l.Href != nil {
+		href = l.Href.String()
+	}
+
+	aux := struct {
+		Href string `json:"href"`
+	}{
+		Href: href,
+	}
+
+	return json.Marshal(aux)
+}
+
+func (l *firmwareUpdateApiResponseEmbeddedFirmwareDataLink) UnmarshalJSON(j []byte) error {
+	var m map[string]interface{}
+
+	err := json.Unmarshal(j, &m)
+	if err != nil {
+		return err
+	}
+
+	if href := m["href"]; href != nil {
+		url, err := url.Parse(href.(string))
+		if err != nil {
+			return err
+		}
+
+		l.Href = url
+	}
+
+	return nil
+}
+
+type firmwareUpdateApiResponseEmbeddedFirmwareLinks struct {
+	Data firmwareUpdateApiResponseEmbeddedFirmwareDataLink `json:"data"`
+}
+
+func firmwareUpdateApiFilter(key, value string) string {
+	return fmt.Sprintf("%s~~%s~~%s", "eq", key, value)
+}

fields/main.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"go/format"
 	"io"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -89,8 +90,6 @@ var fileReps = []replacement{
 	{"ApGroups", "APGroup"},
 }
 
-var embedTypes bool
-
 type Resource struct {
 	StructName     string
 	ResourcePath   string
@@ -194,7 +193,6 @@ func usage() {
 func main() {
 	flag.Usage = usage
 
-	noEmbeddedTypesFlag := flag.Bool("no-embedded-types", true, "Whether to generate top-level type definitions for embedded type definitions")
 	versionBaseDirFlag := flag.String("version-base-dir", ".", "The base directory for version JSON files")
 	outputDirFlag := flag.String("output-dir", ".", "The output directory of the generated Go code")
 	downloadOnly := flag.Bool("download-only", false, "Only download and build the fields JSON directory, do not generate")
@@ -202,8 +200,6 @@ func main() {
 
 	flag.Parse()
 
-	embedTypes = !*noEmbeddedTypesFlag
-
 	specifiedVersion := flag.Arg(0)
 	if specifiedVersion != "" && *useLatestVersion {
 		fmt.Print("error: cannot specify version with latest\n\n")
@@ -216,10 +212,11 @@ func main() {
 	}
 
 	var unifiVersion *version.Version
+	var unifiDownloadUrl *url.URL
 	var err error
 
 	if *useLatestVersion {
-		unifiVersion, err = latestUnifiVersion()
+		unifiVersion, unifiDownloadUrl, err = latestUnifiVersion()
 		if err != nil {
 			panic(err)
 		}
@@ -229,6 +226,11 @@ func main() {
 			fmt.Println(err)
 			os.Exit(1)
 		}
+
+		unifiDownloadUrl, err = url.Parse(fmt.Sprintf("https://dl.ui.com/unifi/%s/unifi_sysvinit_all.deb", unifiVersion))
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	wd, err := os.Getwd()
@@ -251,7 +253,7 @@ func main() {
 		}
 
 		// download fields, create
-		jarFile, err := downloadJar(unifiVersion, fieldsDir)
+		jarFile, err := downloadJar(unifiDownloadUrl, fieldsDir)
 		if err != nil {
 			panic(err)
 		}
@@ -586,9 +588,7 @@ func (r *Resource) generateCode() (string, error) {
 	var buf bytes.Buffer
 	writer := io.Writer(&buf)
 
-	tpl := template.Must(template.New("api.go.tmpl").Funcs(template.FuncMap{
+	tpl := template.Must(template.New("api.go.tmpl").Parse(apiGoTemplate))
-		"embedTypes": func() bool { return embedTypes },
-	}).Parse(apiGoTemplate))
 
 	err = tpl.Execute(writer, r)
 	if err != nil {

fields/version.go

@@ -3,61 +3,47 @@ package main
 import (
 	"encoding/json"
 	"net/http"
+	"net/url"
 
 	"github.com/hashicorp/go-version"
 )
 
-var uiDownloadUrl = "https://www.ui.com/download/?platform=unifi"
+func latestUnifiVersion() (*version.Version, *url.URL, error) {
-
+	url, err := url.Parse(firmwareUpdateApi)
-func latestUnifiVersion() (*version.Version, error) {
-	client := &http.Client{}
-
-	req, err := http.NewRequest("GET", uiDownloadUrl, nil)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	req.Header.Add("X-Requested-With", "XMLHttpRequest")
 
+	query := url.Query()
+	query.Add("filter", firmwareUpdateApiFilter("channel", releaseChannel))
+	query.Add("filter", firmwareUpdateApiFilter("product", unifiControllerProduct))
+	url.RawQuery = query.Encode()
+
+	req, err := http.NewRequest(http.MethodGet, url.String(), nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer resp.Body.Close()
 
-	var respData struct {
+	var respData firmwareUpdateApiResponse
-		Downloads []struct {
-			Id           int    `json:"id"`
-			CategorySlug string `json:"category__slug"`
-			Filename     string `json:"filename"`
-			Version      string `json:"version"`
-		} `json:"downloads"`
-	}
-	var latestVersion *version.Version
-
 	err = json.NewDecoder(resp.Body).Decode(&respData)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	for _, download := range respData.Downloads {
+	for _, firmware := range respData.Embedded.Firmware {
-		if download.CategorySlug != "software" {
+		if firmware.Platform != debianPlatform {
 			continue
 		}
 
-		if download.Filename != "unifi_sysvinit_all.deb" {
+		return firmware.Version.Core(), firmware.Links.Data.Href, nil
-			continue
-		}
-
-		downloadVersion, err := version.NewVersion(download.Version)
-		if err != nil {
-			// Skip this entry if the version isn't valid.
-			continue
-		}
-
-		if latestVersion == nil || downloadVersion.GreaterThan(latestVersion) {
-			latestVersion = downloadVersion
-		}
 	}
 
-	return latestVersion, nil
+	return nil, nil, nil
 }

fields/version_test.go

@@ -4,156 +4,87 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
 
 	"github.com/hashicorp/go-version"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestLatestUnifiVersion(t *testing.T) {
-	respData := map[string]interface{}{
+	assert := assert.New(t)
-		"downloads": []map[string]interface{}{
+	require := require.New(t)
-			{
+
-				"architecture":     "",
+	fwVersion, err := version.NewVersion("7.3.83+atag-7.3.83-19645")
-				"build":            "",
+	require.NoError(err)
-				"category__name":   "User Guides",
+
-				"category__slug":   "user-guides",
+	fwDownload, err := url.Parse("https://fw-download.ubnt.com/data/unifi-controller/c31c-debian-7.3.83-c9249c913b91416693b869b9548850c3.deb")
-				"changelog":        "",
+	require.NoError(err)
-				"date_published":   "2018-03-05",
+
-				"description":      "",
+	respData := firmwareUpdateApiResponse{
-				"featured":         true,
+		Embedded: firmwareUpdateApiResponseEmbedded{
-				"file_path":        "/downloads/guides/UniFi/UniFi_Controller_V5_UG.pdf",
+			Firmware: []firmwareUpdateApiResponseEmbeddedFirmware{
-				"filename":         "UniFi_Controller_V5_UG.pdf",
+				{
-				"id":               898,
+					Channel:  releaseChannel,
-				"mib":              "",
+					Created:  "2023-02-06T08:55:31+00:00",
-				"name":             "UniFi® Controller v5 User Guide",
+					Id:       "c9249c91-3b91-4166-93b8-69b9548850c3",
-				"products":         "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-HD|UAP-IW|UAP-LR|UAP-nanoHD|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UC-CK|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-XG-6POE|UWB‑XG|UWB‑XG‑BK",
+					Platform: debianPlatform,
-				"rank":             1,
+					Product:  unifiControllerProduct,
-				"revision_history": "",
+					Version:  fwVersion,
-				"sdk__id":          nil,
+					Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{
-				"size":             nil,
+						Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{
-				"slug":             "unifi-controller-v5-user-guide",
+							Href: fwDownload,
-				"thumbnail":        "https://prd-www-cdn.ubnt.com/media/images/download/user-guides/UniFi_Controller_V5_UG.jpg",
+						},
-				"thumbnail_retina": "https://prd-www-cdn.ubnt.com/media/images/download/user-guides/UniFi_Controller_V5_UG-2x.jpg",
+					},
-				"version":          "",
+				},
-			},
+				{
-			{
+					Channel:  releaseChannel,
-				"architecture":     "",
+					Created:  "2023-02-06T08:51:36+00:00",
-				"build":            "",
+					Id:       "2a600108-7f79-4b3e-b6e0-4dd262460457",
-				"category__name":   "Software",
+					Platform: "document",
-				"category__slug":   "software",
+					Product:  unifiControllerProduct,
-				"changelog":        "https://community.ui.com/releases/UniFi-Network-Controller-6-0-22/910ceffc-f0e9-4518-86c1-df5eeee34695",
+					Version:  fwVersion,
-				"date_published":   "2020-09-17",
+					Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{
-				"description":      "UniFi Network Controller 6.0.22 for Debian/Ubuntu Linux and UniFi Cloud Key.",
+						Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{
-				"featured":         false,
+							Href: nil,
-				"file_path":        "/downloads/unifi/6.0.22/unifi_sysvinit_all.deb",
+						},
-				"filename":         "unifi_sysvinit_all.deb",
+					},
-				"id":               2607,
+				},
-				"mib":              "",
+				{
-				"name":             "UniFi Network Controller 6.0.22 for Debian/Ubuntu Linux and UniFi Cloud Key",
+					Channel:  releaseChannel,
-				"products":         "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USW-16-POE|USW-24-POE|USW-48-POE|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Pro-24-POE|USW-Pro-48-POE|US-XG-6POE|UWB‑XG|UWB‑XG‑BK",
+					Created:  "2023-02-06T08:51:37+00:00",
-				"rank":             350,
+					Id:       "9d2d413d-36ce-4742-a10d-4351aac6f08d",
-				"revision_history": "",
+					Platform: "windows",
-				"sdk__id":          nil,
+					Product:  unifiControllerProduct,
-				"size":             nil,
+					Version:  fwVersion,
-				"slug":             "unifi-network-controller-6022-debianubuntu-linux-and-unifi-cloud-key",
+					Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{
-				"thumbnail":        nil,
+						Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{
-				"thumbnail_retina": nil,
+							Href: nil,
-				"version":          "6.0.22",
+						},
-			},
+					},
-			{
+				},
-				"architecture":     "",
-				"build":            "",
-				"category__name":   "Software",
-				"category__slug":   "software",
-				"changelog":        "https://community.ui.com/releases/0cffd3ed-7429-4529-9a20-9fead78ebf66",
-				"date_published":   "2021-03-25",
-				"description":      "UniFi Network Controller 6.1.71 for Debian/Ubuntu Linux and UniFi Cloud Key.",
-				"featured":         false,
-				"file_path":        "/downloads/unifi/6.1.71/unifi_sysvinit_all.deb",
-				"filename":         "unifi_sysvinit_all.deb",
-				"id":               2777,
-				"mib":              "",
-				"name":             "UniFi Network Controller 6.1.71 for Debian/Ubuntu Linux and UniFi Cloud Key",
-				"products":         "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USP-RPS|USW-16-POE|USW-24|USW-24-POE|USW-48|USW-48-POE|USW-Aggregation|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Lite-8-PoE|USW-Pro-24|USW-Pro-24-POE|USW-Pro-48|USW-Pro-48-POE|US-XG-6POE|UWB‑XG|UWB‑XG‑BK",
-				"rank":             423,
-				"revision_history": "",
-				"sdk__id":          nil,
-				"size":             nil,
-				"slug":             "unifi-network-controller-6171-debianubuntu-linux-and-unifi-cloud-key",
-				"thumbnail":        nil,
-				"thumbnail_retina": nil,
-				"version":          "6.1.71",
-			},
-			{
-				"architecture":     "",
-				"build":            "",
-				"category__name":   "Software",
-				"category__slug":   "software",
-				"changelog":        "https://community.ui.com/releases/0dfcbc77-8a4f-4e20-bb93-07bbb0237e3a",
-				"date_published":   "2021-06-21",
-				"description":      "UniFi Network Application 6.2.26 for Debian/Ubuntu Linux and UniFi Cloud Key.",
-				"featured":         true,
-				"file_path":        "/downloads/unifi/6.2.26/unifi_sysvinit_all.deb",
-				"filename":         "unifi_sysvinit_all.deb",
-				"id":               2840,
-				"mib":              "",
-				"name":             "UniFi Network Application 6.2.26 for Debian/Ubuntu Linux and UniFi Cloud Key",
-				"products":         "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|UDM|UDM-Pro|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USP-RPS|USW-16-POE|USW-24|USW-24-POE|USW-48|USW-48-POE|USW-Aggregation|USW-Enterprise-24-PoE|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Lite-8-PoE|USW-Pro-24|USW-Pro-24-POE|USW-Pro-48|USW-Pro-48-POE|USW-Pro-Aggregation|US-XG-6POE|UWB‑XG|UWB‑XG‑BK",
-				"rank":             440,
-				"revision_history": "",
-				"sdk__id":          nil,
-				"size":             nil,
-				"slug":             "unifi-network-application-6226-debianubuntu-linux-and-unifi-cloud-key",
-				"thumbnail":        nil,
-				"thumbnail_retina": nil,
-				"version":          "6.2.26",
-			},
-			{
-				"architecture":     "",
-				"build":            "",
-				"category__name":   "Firmware",
-				"category__slug":   "firmware",
-				"changelog":        "https://community.ui.com/releases/a98a71d1-ce1e-4823-a1d2-4a5fa3d642b9",
-				"date_published":   "2021-07-14",
-				"description":      "UniFi firmware 5.60.9 for U6-Lite",
-				"featured":         true,
-				"file_path":        "/downloads/unifi/firmware/UAL6/5.60.9.12980/BZ.mt7621_5.60.9+12980.210702.0701.bin",
-				"filename":         "BZ.mt7621_5.60.9+12980.210702.0701.bin",
-				"id":               2847,
-				"mib":              "",
-				"name":             "UniFi firmware 5.60.9 for U6-Lite",
-				"products":         "U6-Lite",
-				"rank":             444,
-				"revision_history": "",
-				"sdk__id":          nil,
-				"size":             nil,
-				"slug":             "unifi-firmware-5609-u6-lite",
-				"thumbnail":        nil,
-				"thumbnail_retina": nil,
-				"version":          "5.60.9",
 			},
 		},
-		"products": []map[string]interface{}{},
 	}
 
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		assert.Equal(t, []string{"XMLHttpRequest"}, req.Header["X-Requested-With"])
+		query := req.URL.Query()
+		assert.Contains(query["filter"], firmwareUpdateApiFilter("channel", releaseChannel))
+		assert.Contains(query["filter"], firmwareUpdateApiFilter("product", unifiControllerProduct))
 
 		resp, err := json.Marshal(respData)
-		assert.Nil(t, err)
+		require.NoError(err)
 
 		_, err = rw.Write(resp)
-		assert.Nil(t, err)
+		require.NoError(err)
 	}))
 	defer server.Close()
 
-	expected, err := version.NewVersion("6.2.26")
+	firmwareUpdateApi = server.URL
-	assert.Nil(t, err)
+	gotVersion, gotDownload, err := latestUnifiVersion()
+	require.NoError(err)
 
-	uiDownloadUrl = server.URL
+	assert.Equal(fwVersion.Core(), gotVersion)
-	actual, err := latestUnifiVersion()
+	assert.Equal(fwDownload, gotDownload)
-	assert.Nil(t, err)
-
-	assert.Equal(t, expected, actual)
 }