From f697d1a33e4e7cefb4164ec977c38ccc2a228099 Mon Sep 17 00:00:00 2001
From: Jianfeng Tan <henry.tjf@antfin.com>
Date: Tue, 8 Oct 2019 17:43:01 +0000
Subject: [PATCH] gofer: reduce CPU usage on GC as of frequent readdir

Refer to golang mallocgc(), each time of allocating an object > 32 KB,
a gc will be triggered.

When we do readdir, sentry always passes 65535, which leads to a malloc
of 65535 * sizeof(p9.Direnta) > 32 KB.

Considering we already use slice append, let's avoid defining the
capability for this slide.

Command for test:

Before this change:

  (container)$ time tree linux-5.3.1 > /dev/null

  real    0m54.272s
  user    0m2.010s
  sys     0m1.740s
  (CPU usage of Gofer: ~30 cores)

  (host)$ perf top -p <pid-of-gofer>

    42.57%  runsc        [.] runtime.gcDrain
    23.41%  runsc        [.] runtime.(*lfstack).pop
     9.74%  runsc        [.] runtime.greyobject
     8.06%  runsc        [.] runtime.(*lfstack).push
     4.33%  runsc        [.] runtime.scanobject
     1.69%  runsc        [.] runtime.findObject
     1.12%  runsc        [.] runtime.findrunnable
     0.69%  runsc        [.] runtime.runqgrab
    ...

  (host)$ mkdir test && cd test
  (host)$ for i in `seq 1 65536`; do mkdir $i; done
  (container)$ time ls test/ > /dev/null

  real    2m10.934s
  user    0m0.280s
  sys     0m4.260s
  (CPU usage of Gofer: ~1 core)

After this change:

  (container)$ time tree linux-5.3.1 > /dev/null

  real    0m22.465s
  user    0m1.270s
  sys     0m1.310s
  (CPU usage of Gofer: ~1 core)

  $ perf top -p <pid-of-gofer>

    20.57%  runsc        [.] runtime.gcDrain
     7.15%  runsc        [.] runtime.(*lfstack).pop
     4.11%  runsc        [.] runtime.scanobject
     3.78%  runsc        [.] runtime.greyobject
     2.78%  runsc        [.] runtime.(*lfstack).push
    ...

  (host)$ mkdir test && cd test
  (host)$ for i in `seq 1 65536`; do mkdir $i; done
  (container)$ time ls test/ > /dev/null

  real    0m13.338s
  user    0m0.190s
  sys     0m3.980s
  (CPU usage of Gofer: ~0.8 core)

Fixes #898

Signed-off-by: Jianfeng Tan <henry.tjf@antfin.com>
---
 runsc/fsgofer/fsgofer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runsc/fsgofer/fsgofer.go b/runsc/fsgofer/fsgofer.go
index 9117d9616..c9add64ec 100644
--- a/runsc/fsgofer/fsgofer.go
+++ b/runsc/fsgofer/fsgofer.go
@@ -956,14 +956,14 @@ func (l *localFile) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) {
 }
 
 func (l *localFile) readDirent(f int, offset uint64, count uint32, skip uint64) ([]p9.Dirent, error) {
+	var dirents []p9.Dirent
+
 	// Limit 'count' to cap the slice size that is returned.
 	const maxCount = 100000
 	if count > maxCount {
 		count = maxCount
 	}
 
-	dirents := make([]p9.Dirent, 0, count)
-
 	// Pre-allocate buffers that will be reused to get partial results.
 	direntsBuf := make([]byte, 8192)
 	names := make([]string, 0, 100)