From 6714bee53bbe6cdc3943e59f30ec7765907e80b0 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Wed, 3 Jul 2019 00:38:04 -0400 Subject: [PATCH] Add generate-syscall-docs command. Adds a new command to automatically generate the syscall compatibility docs from runsc. --- .gcloudignore | 2 + .gitignore | 1 + Makefile | 20 +- cloudbuild.yaml | 55 +- cmd/generate-syscall-docs/main.go | 203 +++++ cmd/gvisor-website/main.go | 4 +- .../docs/user_guide/compatibility/amd64.md | 713 ------------------ .../user_guide/compatibility/linux/.gitignore | 1 + go.mod | 3 + 9 files changed, 279 insertions(+), 723 deletions(-) create mode 100644 cmd/generate-syscall-docs/main.go delete mode 100644 content/docs/user_guide/compatibility/amd64.md create mode 100644 content/docs/user_guide/compatibility/linux/.gitignore create mode 100644 go.mod diff --git a/.gcloudignore b/.gcloudignore index a1ac4bdf6..ec372978f 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -1,3 +1,5 @@ +bin/ public/ resources/ node_modules/ +upstream/ diff --git a/.gitignore b/.gitignore index 56919aaf9..ec372978f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +bin/ public/ resources/ node_modules/ diff --git a/Makefile b/Makefile index 7415fc653..af41e99d5 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ GCLOUD := gcloud GCP_PROJECT := gvisor-website # Source Go files, example: main.go, foo/bar.go. +GEN_SOURCE = $(wildcard cmd/generate-syscall-docs/*) APP_SOURCE = $(wildcard cmd/gvisor-website/*) # Target Go files, example: public/main.go, public/foo/bar.go. APP_TARGET = $(patsubst cmd/gvisor-website/%,public/%,$(APP_SOURCE)) @@ -40,7 +41,7 @@ content/docs/community/sigs: upstream/community $(wildcard upstream/community/si $(APP_TARGET): public $(APP_SOURCE) cp -a cmd/gvisor-website/$(patsubst public/%,%,$@) public/ -public/static: node_modules config.toml $(shell find archetypes assets content themes -type f | sed 's/ /\\ /g') +public/static: compatibility-docs node_modules config.toml $(shell find archetypes assets content themes -type f | sed 's/ /\\ /g') HUGO_ENV="production" $(HUGO) node_modules: package.json package-lock.json @@ -48,8 +49,19 @@ node_modules: package.json package-lock.json # See: https://github.com/npm/npm/issues/18286 $(NPM) ci +upstream/gvisor/bazel-bin/runsc/linux_amd64_pure_stripped/runsc: upstream-gvisor + cd upstream/gvisor && bazel build runsc + +bin/generate-syscall-docs: $(GEN_SOURCE) + mkdir -p bin/ + go build -o bin/generate-syscall-docs gvisor.dev/website/cmd/generate-syscall-docs + +.PHONY: compatibility-docs +compatibility-docs: bin/generate-syscall-docs upstream/gvisor/bazel-bin/runsc/linux_amd64_pure_stripped/runsc + ./upstream/gvisor/bazel-bin/runsc/linux_amd64_pure_stripped/runsc help syscalls -o json | ./bin/generate-syscall-docs -out ./content/docs/user_guide/compatibility/ + # Run a local content development server. Redirects will not be supported. -server: all-upstream +server: all-upstream compatibility-docs $(HUGO) server -FD --port 8080 .PHONY: server @@ -63,7 +75,7 @@ deploy: $(APP_TARGET) # Submit a build to Cloud Build manually. Used to test cloudbuild.yaml changes. cloud-build: - gcloud builds submit --config cloudbuild/cloudbuild.yaml . + gcloud builds submit --config cloudbuild.yaml . # Build and push the hugo Docker image used by Cloud Build. hugo-docker-image: @@ -78,5 +90,5 @@ htmlproofer-docker-image: .PHONY: htmlproofer-docker-image clean: - rm -rf public/ resources/ node_modules/ upstream/ + rm -rf public/ resources/ node_modules/ upstream/ content/docs/user_guide/compatibility/linux/ .PHONY: clean diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 25944303a..f8bf38566 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,15 +13,62 @@ # limitations under the License. steps: + - name: 'bash' + args: ['bash', '-c', 'mkdir -p upstream/'] + # Clone the upstream repos + - name: 'gcr.io/cloud-builders/git' + args: ['clone', 'https://github.com/google/gvisor.git'] + dir: 'upstream' + - name: 'gcr.io/cloud-builders/git' + args: ['clone', 'https://gvisor.googlesource.com/community'] + dir: 'upstream' + # Build runsc + - name: 'gcr.io/cloud-builders/bazel' + args: + - 'build' + - '--show_timestamps' + - '--test_output=errors' + - '--keep_going' + - '--verbose_failures' + - 'runsc' + dir: 'upstream/gvisor' + # Build the compatibility doc generator tool + - name: 'golang' + env: ['GO111MODULE=on'] + args: + - 'go' + - 'build' + - '-o' + - 'bin/generate-syscall-docs' + - 'gvisor.dev/website/cmd/generate-syscall-docs' + # Generate compatibility docs. + - name: 'bash' + args: + - 'bash' + - '-c' + - > + ./upstream/gvisor/bazel-bin/runsc/linux_amd64_pure_stripped/runsc help syscalls -o json | + ./bin/generate-syscall-docs -out ./content/docs/user_guide/compatibility/ + # Pull npm dependencies for scss + - name: 'gcr.io/cloud-builders/npm' + args: ['ci'] # Generate the website. - name: 'gcr.io/gvisor-website/hugo:0.53' - args: ["make"] + env: ['HUGO_ENV=production'] + args: ["hugo"] # Test the HTML for issues. - name: 'gcr.io/gvisor-website/html-proofer:3.10.2' - args: ["htmlproofer", "--disable-external", "--check-html", "public/static"] + args: + - "htmlproofer" + - "--disable-external" + - "--check-html" + - "public/static" # Deploy to App Engine only for master branch. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' - args: + args: - '-c' - - 'if [[ "$PROJECT_ID" == "gvisor-website" && "$BRANCH_NAME" == "master" ]]; then gcloud app deploy public/app.yaml; fi' + - > + if [[ "$PROJECT_ID" == "gvisor-website" && "$BRANCH_NAME" == "master" ]]; then + gcloud app deploy public/app.yaml; + fi diff --git a/cmd/generate-syscall-docs/main.go b/cmd/generate-syscall-docs/main.go new file mode 100644 index 000000000..006bd1ce5 --- /dev/null +++ b/cmd/generate-syscall-docs/main.go @@ -0,0 +1,203 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "text/template" +) + +type CompatibilityInfo map[string]map[string]ArchInfo + +// ArchInfo is compatbility doc for an architecture. +type ArchInfo struct { + // Syscalls maps syscall number for the architecture to the doc. + Syscalls map[uintptr]SyscallDoc `json:"syscalls"` +} + +// SyscallDoc represents a single item of syscall documentation. +type SyscallDoc struct { + Name string `json:"name"` + Support string `json:"support"` + Note string `json:"note,omitempty"` + URLs []string `json:"urls,omitempty"` +} + +var mdTemplate = template.Must(template.New("out").Parse(` ++++ +title = "{{.OS}}/{{.Arch}}" +description = "Syscall Compatibility Reference Documentation for {{.OS}}/{{.Arch}}" +weight = {{.Weight}} ++++ + +This table is a reference of {{.OS}} syscalls for the {{.Arch}} architecture and +their compatibility status in gVisor. gVisor does not support all syscalls and +some syscalls may have a partial implementation. + +Of {{.Total}} syscalls, {{.Supported}} syscalls have a full or partial +implementation. There are currently {{.Unsupported}} unsupported +syscalls. {{if .Undocumented}}{{.Undocumented}} syscalls are not yet documented.{{end}} + + + + + + + + + + + + {{range $i, $syscall := .Syscalls}} + + + + + + + {{end}} + +
#NameSupportNotes
{{.Number}}{{.Name}}{{.Support}}{{.Note}} {{range $i, $url := .URLs}}
See: {{.}}{{end}}
+`)) + +// Fatalf writes a message to stderr and exits with error code 1 +func Fatalf(format string, a ...interface{}) { + fmt.Fprintf(os.Stderr, format, a...) + os.Exit(1) +} + +func main() { + inputFlag := flag.String("in", "-", "File to input ('-' for stdin)") + outputDir := flag.String("out", ".", "Directory to output files.") + + flag.Parse() + + var input io.Reader + if *inputFlag == "-" { + input = os.Stdin + } else { + i, err := os.Open(*inputFlag) + if err != nil { + Fatalf("Error opening %q: %v", *inputFlag, err) + } + input = i + } + input = bufio.NewReader(input) + + var info CompatibilityInfo + d := json.NewDecoder(input) + if err := d.Decode(&info); err != nil { + Fatalf("Error reading json: %v") + } + + weight := 0 + for osName, osInfo := range info { + for archName, archInfo := range osInfo { + outDir := filepath.Join(*outputDir, osName) + outFile := filepath.Join(outDir, archName+".md") + + if err := os.MkdirAll(outDir, 0755); err != nil { + Fatalf("Error creating directory %q: %v", *outputDir, err) + } + + f, err := os.OpenFile(outFile, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + Fatalf("Error opening file %q: %v", outFile, err) + } + defer f.Close() + + weight += 10 + data := struct { + OS string + Arch string + Weight int + Total int + Supported int + Unsupported int + Undocumented int + Syscalls []struct { + Name string + Number uintptr + Support string + Note string + URLs []string + } + }{ + OS: strings.Title(osName), + Arch: archName, + Weight: weight, + Total: 0, + Supported: 0, + Unsupported: 0, + Undocumented: 0, + Syscalls: []struct { + Name string + Number uintptr + Support string + Note string + URLs []string + }{}, + } + + for num, s := range archInfo.Syscalls { + switch s.Support { + case "Full Support", "Partial Support": + data.Supported++ + case "Unimplemented": + data.Unsupported++ + default: + data.Undocumented++ + } + data.Total++ + + for i := range s.URLs { + if !strings.HasPrefix(s.URLs[i], "http://") && !strings.HasPrefix(s.URLs[i], "https://") { + s.URLs[i] = "https://" + s.URLs[i] + } + } + + data.Syscalls = append(data.Syscalls, struct { + Name string + Number uintptr + Support string + Note string + URLs []string + }{ + Name: s.Name, + Number: num, + Support: s.Support, + Note: s.Note, // TODO urls + URLs: s.URLs, + }) + } + + sort.Slice(data.Syscalls, func(i, j int) bool { + return data.Syscalls[i].Number < data.Syscalls[j].Number + }) + + if err := mdTemplate.Execute(f, data); err != nil { + Fatalf("Error writing file %q: %v", outFile, err) + } + } + } +} diff --git a/cmd/gvisor-website/main.go b/cmd/gvisor-website/main.go index ecae64279..dd6058081 100644 --- a/cmd/gvisor-website/main.go +++ b/cmd/gvisor-website/main.go @@ -37,7 +37,7 @@ var redirects = map[string]string{ // Redirects to compatibility docs. "/c": "/docs/user_guide/compatibility", - "/c/linux/amd64": "/docs/user_guide/compatibility/amd64", + "/c/linux/amd64": "/docs/user_guide/compatibility/linux/amd64", // Deprecated, but links continue to work. "/cl": "https://gvisor-review.googlesource.com", @@ -49,7 +49,7 @@ var prefixHelpers = map[string]string{ "pr": "https://github.com/google/gvisor/pull/%s", // Redirects to compatibility docs. - "c/linux/amd64": "/docs/user_guide/compatibility/amd64/#%s", + "c/linux/amd64": "/docs/user_guide/compatibility/linux/amd64/#%s", // Deprecated, but links continue to work. "cl": "https://gvisor-review.googlesource.com/c/gvisor/+/%s", diff --git a/content/docs/user_guide/compatibility/amd64.md b/content/docs/user_guide/compatibility/amd64.md deleted file mode 100644 index b9500bcc6..000000000 --- a/content/docs/user_guide/compatibility/amd64.md +++ /dev/null @@ -1,713 +0,0 @@ -+++ -title = "AMD64" -description = "Syscall Compatibility Reference Documentation for AMD64" -weight = 10 -+++ - -This table is a reference of Linux syscalls for the AMD64 architecture and -their compatibility status in gVisor. gVisor does not support all syscalls and -some syscalls may have a partial implementation. - -Of 329 syscalls, 47 syscalls have a full or partial -implementation. There are currently 51 unimplemented -syscalls. 231 syscalls are not yet documented. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#NameSupportGitHub IssueNotes
68msggetUnimplementedReturns ENOSYS
69msgsndUnimplementedReturns ENOSYS
70msgrcvUnimplementedReturns ENOSYS
71msgctlUnimplementedReturns ENOSYS
122setfsuidUnimplementedReturns ENOSYS
123setfsgidUnimplementedReturns ENOSYS
134uselibUnimplementedReturns ENOSYS; Obsolete
135personalityPartialReturns EINVAL; Unable to change personality
136ustatUnimplementedReturns ENOSYS; Needs filesystem support
139sysfsUnimplementedReturns ENOSYS
142sched_setparamPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_nice; ENOSYS otherwise
148sched_rr_get_intervalPartialReturns EPERM
153vhangupPartialReturns EPERM
154modify_ldtPartialReturns EPERM
155pivot_rootPartialReturns EPERM
156sysctlPartialReturns EPERM
159adjtimexPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_time; ENOSYS otherwise
163acctPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_pacct; ENOSYS otherwise
164settimeofdayPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_time; ENOSYS otherwise
167swaponPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
168swapoffPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
169rebootPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_boot; ENOSYS otherwise
172ioplPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_rawio; ENOSYS otherwise
173iopermPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_rawio; ENOSYS otherwise
174create_modulePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_module; ENOSYS otherwise
175init_modulePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_module; ENOSYS otherwise
176delete_modulePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_module; ENOSYS otherwise
177get_kernel_symsUnimplementedReturns ENOSYS; Not supported in > 2.6
178query_moduleUnimplementedReturns ENOSYS; Not supported in > 2.6
179quotactlPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
180nfsservctlUnimplementedReturns ENOSYS; Does not exist > 3.1
181getpmsgUnimplementedReturns ENOSYS; Not implemented in Linux
182putpmsgUnimplementedReturns ENOSYS; Not implemented in Linux
183afs_syscallUnimplementedReturns ENOSYS; Not implemented in Linux
184tuxcallUnimplementedReturns ENOSYS; Not implemented in Linux
185securityUnimplementedReturns ENOSYS; Not implemented in Linux
187readaheadUnimplementedReturns ENOSYS
188setxattrPartialReturns ENOTSUP; Requires filesystem support
189lsetxattrPartialReturns ENOTSUP; Requires filesystem support
190fsetxattrPartialReturns ENOTSUP; Requires filesystem support
191getxattrPartialReturns ENOTSUP; Requires filesystem support
192lgetxattrPartialReturns ENOTSUP; Requires filesystem support
193fgetxattrPartialReturns ENOTSUP; Requires filesystem support
194listxattrPartialReturns ENOTSUP; Requires filesystem support
195llistxattrPartialReturns ENOTSUP; Requires filesystem support
196flistxattrPartialReturns ENOTSUP; Requires filesystem support
197removexattrPartialReturns ENOTSUP; Requires filesystem support
198lremovexattrPartialReturns ENOTSUP; Requires filesystem support
199fremovexattrPartialReturns ENOTSUP; Requires filesystem support
205set_thread_areaUnimplementedReturns ENOSYS; Expected to return ENOSYS on 64-bit
211get_thread_areaUnimplementedReturns ENOSYS; Expected to return ENOSYS on 64-bit
212lookup_dcookiePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
214epoll_ctl_oldUnimplementedReturns ENOSYS; Deprecated
215epoll_wait_oldUnimplementedReturns ENOSYS; Deprecated
216remap_file_pagesUnimplementedReturns ENOSYS; Deprecated
220semtimedopUnimplementedReturns ENOSYS
236vserverUnimplementedReturns ENOSYS; Not implemented by Linux
237mbindPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_nice; ENOSYS otherwise
240mq_openUnimplementedReturns ENOSYS
241mq_unlinkUnimplementedReturns ENOSYS
242mq_timedsendUnimplementedReturns ENOSYS
243mq_timedreceiveUnimplementedReturns ENOSYS
244mq_notifyUnimplementedReturns ENOSYS
245mq_getsetattrUnimplementedReturns ENOSYS
248add_keyPartialReturns EACCES; Not available to user
249request_keyPartialReturns EACCES; Not available to user
250keyctlPartialReturns EACCES; Not available to user
251ioprio_setPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
252ioprio_getPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_admin; ENOSYS otherwise
256migrate_pagesPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_nice; ENOSYS otherwise
273set_robust_listUnimplementedReturns ENOSYS; Obsolete
274get_robust_listUnimplementedReturns ENOSYS; Obsolete
275spliceUnimplementedReturns ENOSYS
276teeUnimplementedReturns ENOSYS
278vmspliceUnimplementedReturns ENOSYS
279move_pagesPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_nice; ENOSYS otherwise
282signalfdUnimplementedReturns ENOSYS
289signalfd4UnimplementedReturns ENOSYS
298perf_event_openPartialReturns ENODEV; No support for perf counters
300fanotify_initUnimplementedReturns ENOSYS; Needs CONFIG_FANOTIFY
301fanotify_markUnimplementedReturns ENOSYS; Needs CONFIG_FANOTIFY
303name_to_handle_atPartialReturns EOPNOTSUPP; Needs filesystem support
304open_by_handle_atPartialReturns EOPNOTSUPP; Needs filesystem support
305clock_adjtimePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_module; ENOSYS otherwise
308setnsUnimplementedReturns ENOSYS
310process_vm_readvUnimplementedReturns ENOSYS
311process_vm_writevUnimplementedReturns ENOSYS
312kcmpPartialReturns EPERM or ENOSYS; Requires cap_sys_ptrace
313finit_modulePartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_module; ENOSYS otherwise
314sched_setattrUnimplementedReturns ENOSYS
315sched_getattrUnimplementedReturns ENOSYS
316renameat2UnimplementedReturns ENOSYS
319memfd_createUnimplementedReturns ENOSYS
321bpfPartialReturns EPERM or ENOSYS; Returns EPERM if the process does not have cap_sys_boot; ENOSYS otherwise
322execveatUnimplementedReturns ENOSYS
323userfaultfdUnimplementedReturns ENOSYS
324membarrierUnimplementedReturns ENOSYS
326copy_file_rangeUnimplementedReturns ENOSYS
diff --git a/content/docs/user_guide/compatibility/linux/.gitignore b/content/docs/user_guide/compatibility/linux/.gitignore new file mode 100644 index 000000000..97958c35e --- /dev/null +++ b/content/docs/user_guide/compatibility/linux/.gitignore @@ -0,0 +1 @@ +amd64.md diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..e58df1f2c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gvisor.dev/website + +go 1.12