From 97b562a6d2ae81864cdd5e5c2490e788c37d795f Mon Sep 17 00:00:00 2001 From: Joshua Spence Date: Fri, 26 May 2023 15:29:07 +1000 Subject: [PATCH] Improvements to field generation (#137) * Remove `no-embedded-types` flag We don't use this. * Use `fw-download.ubnt.com` * Return firmware download URL from `latestUnifiVersion` * Don't use version metadata --- fields/api.go.tmpl | 14 --- fields/extract.go | 8 +- fields/fwupdate.go | 82 +++++++++++++++++ fields/main.go | 20 ++--- fields/version.go | 58 +++++------- fields/version_test.go | 195 +++++++++++++---------------------------- 6 files changed, 180 insertions(+), 197 deletions(-) create mode 100644 fields/fwupdate.go diff --git a/fields/api.go.tmpl b/fields/api.go.tmpl index 4a4fd6a..d0dadeb 100644 --- a/fields/api.go.tmpl +++ b/fields/api.go.tmpl @@ -17,11 +17,6 @@ {{- else }} dst.{{ .FieldName }} = {{ .FieldType }}(aux.{{ .FieldName }}) {{- end }}{{- end }}{{- end }} -{{ define "field-embed" }} - {{ .FieldName }} {{ if .IsArray }}[]{{end}}{{ if not .Fields }}{{ .FieldType }}{{ else }}struct { - {{ range $fk, $fv := .Fields }}{{ if not $fv }} - {{ else }}{{- template "field-embed" $fv }}{{ end }}{{ end }} -}{{ end }} `json:"{{ .JSONName }}{{ if .OmitEmpty }},omitempty{{ end }}"` {{ if .FieldValidation }}// {{ .FieldValidation }}{{ end }} {{- end }} // Code generated from ace.jar fields *.json files // DO NOT EDIT. @@ -40,14 +35,6 @@ var ( _ json.Marshaler ) -{{ if embedTypes -}} -{{- $k := .StructName -}} -{{- $v :=index .Types .StructName -}} -type {{ $k }} struct { - {{ range $fk, $fv := $v.Fields }}{{ if not $fv }} - {{ else }}{{- template "field-embed" $fv }}{{ end }}{{ end }} -} -{{- else -}} {{ range $k, $v := .Types }} type {{ $k }} struct { {{ range $fk, $fv := $v.Fields }}{{ if not $fv }} @@ -76,7 +63,6 @@ func (dst *{{ $k }}) UnmarshalJSON(b []byte) error { return nil } {{ end }} -{{- end -}} {{ if not .IsSetting }} func (c *Client) list{{ .StructName }}(ctx context.Context, site string) ([]{{ .StructName }}, error) { diff --git a/fields/extract.go b/fields/extract.go index 3c5a9f2..451df5e 100644 --- a/fields/extract.go +++ b/fields/extract.go @@ -8,21 +8,19 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path" "path/filepath" "strings" - "github.com/hashicorp/go-version" "github.com/iancoleman/strcase" "github.com/ulikunitz/xz" "github.com/xor-gate/ar" ) -func downloadJar(version *version.Version, outputDir string) (string, error) { - url := fmt.Sprintf("https://dl.ui.com/unifi/%s/unifi_sysvinit_all.deb", version) - - debResp, err := http.Get(url) +func downloadJar(url *url.URL, outputDir string) (string, error) { + debResp, err := http.Get(url.String()) if err != nil { return "", fmt.Errorf("unable to download deb: %w", err) } diff --git a/fields/fwupdate.go b/fields/fwupdate.go new file mode 100644 index 0000000..404c054 --- /dev/null +++ b/fields/fwupdate.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/hashicorp/go-version" +) + +var firmwareUpdateApi = "https://fw-update.ubnt.com/api/firmware-latest" + +const ( + debianPlatform = "debian" + releaseChannel = "release" + unifiControllerProduct = "unifi-controller" +) + +type firmwareUpdateApiResponse struct { + Embedded firmwareUpdateApiResponseEmbedded `json:"_embedded"` +} + +type firmwareUpdateApiResponseEmbedded struct { + Firmware []firmwareUpdateApiResponseEmbeddedFirmware `json:"firmware"` +} + +type firmwareUpdateApiResponseEmbeddedFirmware struct { + Channel string `json:"channel"` + Created string `json:"created"` + Id string `json:"id"` + Platform string `json:"platform"` + Product string `json:"product"` + Version *version.Version `json:"version"` + Links firmwareUpdateApiResponseEmbeddedFirmwareLinks `json:"_links"` +} + +type firmwareUpdateApiResponseEmbeddedFirmwareDataLink struct { + Href *url.URL `json:"href"` +} + +func (l firmwareUpdateApiResponseEmbeddedFirmwareDataLink) MarshalJSON() ([]byte, error) { + var href string + if l.Href != nil { + href = l.Href.String() + } + + aux := struct { + Href string `json:"href"` + }{ + Href: href, + } + + return json.Marshal(aux) +} + +func (l *firmwareUpdateApiResponseEmbeddedFirmwareDataLink) UnmarshalJSON(j []byte) error { + var m map[string]interface{} + + err := json.Unmarshal(j, &m) + if err != nil { + return err + } + + if href := m["href"]; href != nil { + url, err := url.Parse(href.(string)) + if err != nil { + return err + } + + l.Href = url + } + + return nil +} + +type firmwareUpdateApiResponseEmbeddedFirmwareLinks struct { + Data firmwareUpdateApiResponseEmbeddedFirmwareDataLink `json:"data"` +} + +func firmwareUpdateApiFilter(key, value string) string { + return fmt.Sprintf("%s~~%s~~%s", "eq", key, value) +} diff --git a/fields/main.go b/fields/main.go index 123fd9e..ee5e6eb 100644 --- a/fields/main.go +++ b/fields/main.go @@ -9,6 +9,7 @@ import ( "fmt" "go/format" "io" + "net/url" "os" "path" "path/filepath" @@ -89,8 +90,6 @@ var fileReps = []replacement{ {"ApGroups", "APGroup"}, } -var embedTypes bool - type Resource struct { StructName string ResourcePath string @@ -194,7 +193,6 @@ func usage() { func main() { flag.Usage = usage - noEmbeddedTypesFlag := flag.Bool("no-embedded-types", true, "Whether to generate top-level type definitions for embedded type definitions") versionBaseDirFlag := flag.String("version-base-dir", ".", "The base directory for version JSON files") outputDirFlag := flag.String("output-dir", ".", "The output directory of the generated Go code") downloadOnly := flag.Bool("download-only", false, "Only download and build the fields JSON directory, do not generate") @@ -202,8 +200,6 @@ func main() { flag.Parse() - embedTypes = !*noEmbeddedTypesFlag - specifiedVersion := flag.Arg(0) if specifiedVersion != "" && *useLatestVersion { fmt.Print("error: cannot specify version with latest\n\n") @@ -216,10 +212,11 @@ func main() { } var unifiVersion *version.Version + var unifiDownloadUrl *url.URL var err error if *useLatestVersion { - unifiVersion, err = latestUnifiVersion() + unifiVersion, unifiDownloadUrl, err = latestUnifiVersion() if err != nil { panic(err) } @@ -229,6 +226,11 @@ func main() { fmt.Println(err) os.Exit(1) } + + unifiDownloadUrl, err = url.Parse(fmt.Sprintf("https://dl.ui.com/unifi/%s/unifi_sysvinit_all.deb", unifiVersion)) + if err != nil { + panic(err) + } } wd, err := os.Getwd() @@ -251,7 +253,7 @@ func main() { } // download fields, create - jarFile, err := downloadJar(unifiVersion, fieldsDir) + jarFile, err := downloadJar(unifiDownloadUrl, fieldsDir) if err != nil { panic(err) } @@ -586,9 +588,7 @@ func (r *Resource) generateCode() (string, error) { var buf bytes.Buffer writer := io.Writer(&buf) - tpl := template.Must(template.New("api.go.tmpl").Funcs(template.FuncMap{ - "embedTypes": func() bool { return embedTypes }, - }).Parse(apiGoTemplate)) + tpl := template.Must(template.New("api.go.tmpl").Parse(apiGoTemplate)) err = tpl.Execute(writer, r) if err != nil { diff --git a/fields/version.go b/fields/version.go index 5d75987..88080d3 100644 --- a/fields/version.go +++ b/fields/version.go @@ -3,61 +3,47 @@ package main import ( "encoding/json" "net/http" + "net/url" "github.com/hashicorp/go-version" ) -var uiDownloadUrl = "https://www.ui.com/download/?platform=unifi" - -func latestUnifiVersion() (*version.Version, error) { - client := &http.Client{} - - req, err := http.NewRequest("GET", uiDownloadUrl, nil) +func latestUnifiVersion() (*version.Version, *url.URL, error) { + url, err := url.Parse(firmwareUpdateApi) if err != nil { - return nil, err + return nil, nil, err } - req.Header.Add("X-Requested-With", "XMLHttpRequest") + query := url.Query() + query.Add("filter", firmwareUpdateApiFilter("channel", releaseChannel)) + query.Add("filter", firmwareUpdateApiFilter("product", unifiControllerProduct)) + url.RawQuery = query.Encode() + + req, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + return nil, nil, err + } + + client := &http.Client{} resp, err := client.Do(req) if err != nil { - return nil, err + return nil, nil, err } defer resp.Body.Close() - var respData struct { - Downloads []struct { - Id int `json:"id"` - CategorySlug string `json:"category__slug"` - Filename string `json:"filename"` - Version string `json:"version"` - } `json:"downloads"` - } - var latestVersion *version.Version - + var respData firmwareUpdateApiResponse err = json.NewDecoder(resp.Body).Decode(&respData) if err != nil { - return nil, err + return nil, nil, err } - for _, download := range respData.Downloads { - if download.CategorySlug != "software" { + for _, firmware := range respData.Embedded.Firmware { + if firmware.Platform != debianPlatform { continue } - if download.Filename != "unifi_sysvinit_all.deb" { - continue - } - - downloadVersion, err := version.NewVersion(download.Version) - if err != nil { - // Skip this entry if the version isn't valid. - continue - } - - if latestVersion == nil || downloadVersion.GreaterThan(latestVersion) { - latestVersion = downloadVersion - } + return firmware.Version.Core(), firmware.Links.Data.Href, nil } - return latestVersion, nil + return nil, nil, nil } diff --git a/fields/version_test.go b/fields/version_test.go index f5bd0ed..2a4773a 100644 --- a/fields/version_test.go +++ b/fields/version_test.go @@ -4,156 +4,87 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/url" "testing" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestLatestUnifiVersion(t *testing.T) { - respData := map[string]interface{}{ - "downloads": []map[string]interface{}{ - { - "architecture": "", - "build": "", - "category__name": "User Guides", - "category__slug": "user-guides", - "changelog": "", - "date_published": "2018-03-05", - "description": "", - "featured": true, - "file_path": "/downloads/guides/UniFi/UniFi_Controller_V5_UG.pdf", - "filename": "UniFi_Controller_V5_UG.pdf", - "id": 898, - "mib": "", - "name": "UniFi® Controller v5 User Guide", - "products": "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-HD|UAP-IW|UAP-LR|UAP-nanoHD|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UC-CK|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-XG-6POE|UWB‑XG|UWB‑XG‑BK", - "rank": 1, - "revision_history": "", - "sdk__id": nil, - "size": nil, - "slug": "unifi-controller-v5-user-guide", - "thumbnail": "https://prd-www-cdn.ubnt.com/media/images/download/user-guides/UniFi_Controller_V5_UG.jpg", - "thumbnail_retina": "https://prd-www-cdn.ubnt.com/media/images/download/user-guides/UniFi_Controller_V5_UG-2x.jpg", - "version": "", - }, - { - "architecture": "", - "build": "", - "category__name": "Software", - "category__slug": "software", - "changelog": "https://community.ui.com/releases/UniFi-Network-Controller-6-0-22/910ceffc-f0e9-4518-86c1-df5eeee34695", - "date_published": "2020-09-17", - "description": "UniFi Network Controller 6.0.22 for Debian/Ubuntu Linux and UniFi Cloud Key.", - "featured": false, - "file_path": "/downloads/unifi/6.0.22/unifi_sysvinit_all.deb", - "filename": "unifi_sysvinit_all.deb", - "id": 2607, - "mib": "", - "name": "UniFi Network Controller 6.0.22 for Debian/Ubuntu Linux and UniFi Cloud Key", - "products": "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USW-16-POE|USW-24-POE|USW-48-POE|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Pro-24-POE|USW-Pro-48-POE|US-XG-6POE|UWB‑XG|UWB‑XG‑BK", - "rank": 350, - "revision_history": "", - "sdk__id": nil, - "size": nil, - "slug": "unifi-network-controller-6022-debianubuntu-linux-and-unifi-cloud-key", - "thumbnail": nil, - "thumbnail_retina": nil, - "version": "6.0.22", - }, - { - "architecture": "", - "build": "", - "category__name": "Software", - "category__slug": "software", - "changelog": "https://community.ui.com/releases/0cffd3ed-7429-4529-9a20-9fead78ebf66", - "date_published": "2021-03-25", - "description": "UniFi Network Controller 6.1.71 for Debian/Ubuntu Linux and UniFi Cloud Key.", - "featured": false, - "file_path": "/downloads/unifi/6.1.71/unifi_sysvinit_all.deb", - "filename": "unifi_sysvinit_all.deb", - "id": 2777, - "mib": "", - "name": "UniFi Network Controller 6.1.71 for Debian/Ubuntu Linux and UniFi Cloud Key", - "products": "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USP-RPS|USW-16-POE|USW-24|USW-24-POE|USW-48|USW-48-POE|USW-Aggregation|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Lite-8-PoE|USW-Pro-24|USW-Pro-24-POE|USW-Pro-48|USW-Pro-48-POE|US-XG-6POE|UWB‑XG|UWB‑XG‑BK", - "rank": 423, - "revision_history": "", - "sdk__id": nil, - "size": nil, - "slug": "unifi-network-controller-6171-debianubuntu-linux-and-unifi-cloud-key", - "thumbnail": nil, - "thumbnail_retina": nil, - "version": "6.1.71", - }, - { - "architecture": "", - "build": "", - "category__name": "Software", - "category__slug": "software", - "changelog": "https://community.ui.com/releases/0dfcbc77-8a4f-4e20-bb93-07bbb0237e3a", - "date_published": "2021-06-21", - "description": "UniFi Network Application 6.2.26 for Debian/Ubuntu Linux and UniFi Cloud Key.", - "featured": true, - "file_path": "/downloads/unifi/6.2.26/unifi_sysvinit_all.deb", - "filename": "unifi_sysvinit_all.deb", - "id": 2840, - "mib": "", - "name": "UniFi Network Application 6.2.26 for Debian/Ubuntu Linux and UniFi Cloud Key", - "products": "UAP|UAP-AC-EDU|UAP–AC–IW|UAP–AC–IW–PRO|UAP-AC-LITE|UAP-AC-LR|UAP-AC-M|UAP-AC-M-PRO|UAP-AC-PRO|UAP-BeaconHD|UAP-FlexHD|UAP-HD|UAP-IW|UAP-IW-HD|UAP-LR|UAP-nanoHD|UAP-Outdoor|UAP-Outdoor+|UAP-Outdoor5|UAP-PRO|UAP-SHD|UAP‑XG|UAS-XG|UBB|UC-CK|UCK-G2|UCK-G2-PLUS|UDM|UDM-Pro|US-16-150W|US‑16‑XG|US-24|US-24-250W|US-24-500W|US-48|US-48-500W|US-48-750W|US-8|US-8-150W|US-8-60W|USG|USG-PRO-4|USG-XG-8|US-L2-24-POE|US-L2-48-POE|USP-RPS|USW-16-POE|USW-24|USW-24-POE|USW-48|USW-48-POE|USW-Aggregation|USW-Enterprise-24-PoE|USW-Flex|USW-Flex-Mini|USW-Industrial|USW-Lite-16-POE|USW-Lite-8-PoE|USW-Pro-24|USW-Pro-24-POE|USW-Pro-48|USW-Pro-48-POE|USW-Pro-Aggregation|US-XG-6POE|UWB‑XG|UWB‑XG‑BK", - "rank": 440, - "revision_history": "", - "sdk__id": nil, - "size": nil, - "slug": "unifi-network-application-6226-debianubuntu-linux-and-unifi-cloud-key", - "thumbnail": nil, - "thumbnail_retina": nil, - "version": "6.2.26", - }, - { - "architecture": "", - "build": "", - "category__name": "Firmware", - "category__slug": "firmware", - "changelog": "https://community.ui.com/releases/a98a71d1-ce1e-4823-a1d2-4a5fa3d642b9", - "date_published": "2021-07-14", - "description": "UniFi firmware 5.60.9 for U6-Lite", - "featured": true, - "file_path": "/downloads/unifi/firmware/UAL6/5.60.9.12980/BZ.mt7621_5.60.9+12980.210702.0701.bin", - "filename": "BZ.mt7621_5.60.9+12980.210702.0701.bin", - "id": 2847, - "mib": "", - "name": "UniFi firmware 5.60.9 for U6-Lite", - "products": "U6-Lite", - "rank": 444, - "revision_history": "", - "sdk__id": nil, - "size": nil, - "slug": "unifi-firmware-5609-u6-lite", - "thumbnail": nil, - "thumbnail_retina": nil, - "version": "5.60.9", + assert := assert.New(t) + require := require.New(t) + + fwVersion, err := version.NewVersion("7.3.83+atag-7.3.83-19645") + require.NoError(err) + + fwDownload, err := url.Parse("https://fw-download.ubnt.com/data/unifi-controller/c31c-debian-7.3.83-c9249c913b91416693b869b9548850c3.deb") + require.NoError(err) + + respData := firmwareUpdateApiResponse{ + Embedded: firmwareUpdateApiResponseEmbedded{ + Firmware: []firmwareUpdateApiResponseEmbeddedFirmware{ + { + Channel: releaseChannel, + Created: "2023-02-06T08:55:31+00:00", + Id: "c9249c91-3b91-4166-93b8-69b9548850c3", + Platform: debianPlatform, + Product: unifiControllerProduct, + Version: fwVersion, + Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{ + Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{ + Href: fwDownload, + }, + }, + }, + { + Channel: releaseChannel, + Created: "2023-02-06T08:51:36+00:00", + Id: "2a600108-7f79-4b3e-b6e0-4dd262460457", + Platform: "document", + Product: unifiControllerProduct, + Version: fwVersion, + Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{ + Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{ + Href: nil, + }, + }, + }, + { + Channel: releaseChannel, + Created: "2023-02-06T08:51:37+00:00", + Id: "9d2d413d-36ce-4742-a10d-4351aac6f08d", + Platform: "windows", + Product: unifiControllerProduct, + Version: fwVersion, + Links: firmwareUpdateApiResponseEmbeddedFirmwareLinks{ + Data: firmwareUpdateApiResponseEmbeddedFirmwareDataLink{ + Href: nil, + }, + }, + }, }, }, - "products": []map[string]interface{}{}, } server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - assert.Equal(t, []string{"XMLHttpRequest"}, req.Header["X-Requested-With"]) + query := req.URL.Query() + assert.Contains(query["filter"], firmwareUpdateApiFilter("channel", releaseChannel)) + assert.Contains(query["filter"], firmwareUpdateApiFilter("product", unifiControllerProduct)) resp, err := json.Marshal(respData) - assert.Nil(t, err) + require.NoError(err) _, err = rw.Write(resp) - assert.Nil(t, err) + require.NoError(err) })) defer server.Close() - expected, err := version.NewVersion("6.2.26") - assert.Nil(t, err) + firmwareUpdateApi = server.URL + gotVersion, gotDownload, err := latestUnifiVersion() + require.NoError(err) - uiDownloadUrl = server.URL - actual, err := latestUnifiVersion() - assert.Nil(t, err) - - assert.Equal(t, expected, actual) + assert.Equal(fwVersion.Core(), gotVersion) + assert.Equal(fwDownload, gotDownload) }