1
2
3
4
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 4052245BD4284CDD
echo "deb https://repo.iovisor.org/apt/xenial xenial main" | sudo tee /etc/apt/sources.list.d/iovisor.list
sudo apt-get update
sudo apt-get install -y bcc-tools libbcc-examples linux-headers-$(uname -r)
1
export PATH=$PATH:/usr/share/bcc/tools
1
2
3
4
5
6
7
8
# install go
$ wget https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz
$ tar -C /usr/local -xzf go1.12.7.linux-amd64.tar.gz
$ export PATH=$PATH:/usr/local/go/bin
$ export GOPATH=~/go
$ export PATH=~/go/bin:$PATH
$ go get golang.org/x/sys/unix
$ go get github.com/tobert/pcstat/pcstat
1
2
3
4
5
6
root@perf:/home/ykg# pcstat /bin/ls
+---------+----------------+------------+-----------+---------+
| Name    | Size (bytes)   | Pages      | Cached    | Percent |
|---------+----------------+------------+-----------+---------|
| /bin/ls | 133792         | 33         | 33        | 100.000 |
+---------+----------------+------------+-----------+---------+
1
2
3
4
5
6
7
8
9
10
11
root@perf:/home/ykg# dd if=/dev/sda1 of=file bs=1M count=512
512+0 records in
512+0 records out
536870912 bytes (537 MB, 512 MiB) copied, 19.0035 s, 28.3 MB/s
root@perf:/home/ykg# echo 3 > /proc/sys/vm/drop_caches
root@perf:/home/ykg# pcstat file
+-------+----------------+------------+-----------+---------+
| Name  | Size (bytes)   | Pages      | Cached    | Percent |
|-------+----------------+------------+-----------+---------|
| file  | 536870912      | 131072     | 42496     | 032.422 |
+-------+----------------+------------+-----------+---------+
1
2
3
4
5
6
7
8
root@perf:/home/ykg# dd if=file of=/dev/null bs=1M
512+0 records in
512+0 records out
536870912 bytes (537 MB, 512 MiB) copied, 12.9729 s, 41.4 MB/s
root@perf:/home/ykg# dd if=file of=/dev/null bs=1M
512+0 records in
512+0 records out
536870912 bytes (537 MB, 512 MiB) copied, 0.146201 s, 3.7 GB/s
1
2
3
4
5
6
root@perf:/home/ykg# pcstat file
+-------+----------------+------------+-----------+---------+
| Name  | Size (bytes)   | Pages      | Cached    | Percent |
|-------+----------------+------------+-----------+---------|
| file  | 536870912      | 131072     | 131072    | 100.000 |
+-------+----------------+------------+-----------+---------+

EXP2

1
2
3
4
5
6
7
8
9
10
docker run --privileged --name=app -itd feisky/app:io-direct

root@perf:/home/ykg# docker logs app
Reading data from disk /dev/sdb1 with buffer size 33554432
Time used: 1.415684 s to read 33554432 bytes
Time used: 1.441454 s to read 33554432 bytes
Time used: 1.449582 s to read 33554432 bytes
Time used: 1.449696 s to read 33554432 bytes
Time used: 1.449216 s to read 33554432 bytes
root@perf:/home/ykg#
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
22:05:00 Buffers MB: 13 / Cached MB: 760 / Sort: HITS / Order: ascending
PID      UID      CMD              HITS     MISSES   DIRTIES  READ_HIT%  WRITE_HIT%
    1121 root     atopacctd               1        0        0     100.0%       0.0%
    1504 root     python3                 2        0        0     100.0%       0.0%
    4432 root     dockerd                 8        0        2      75.0%       0.0%
    4303 root     cachetop               14        0        0     100.0%       0.0%
    1442 root     python3                23        2        1      88.0%       4.0%
    4701 root     sh                    110        0        0     100.0%       0.0%
    4703 root     sh                    114        0        0     100.0%       0.0%
    4700 root     python3               366        0        0     100.0%       0.0%
    4702 root     python3               366        0        0     100.0%       0.0%
    4702 root     sh                    401        0        0     100.0%       0.0%
    4700 root     sh                    425        0        0     100.0%       0.0%
    4701 root     iptables              441        0        0     100.0%       0.0%
    4703 root     iptables              522        0        0     100.0%       0.0%
    4611 root     app                  1024        0        0     100.0%       0.0%
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
root@perf:/home/ykg# strace -p $(pgrep app)
strace: Process 4611 attached
write(1, "Time used: 1.450181 s to read 33"..., 45) = 45
close(4)                                = 0
munmap(0x7f46451db000, 33558528)        = 0
nanosleep({tv_sec=1, tv_nsec=0}, 0x7ffed794fdb0) = 0
openat(AT_FDCWD, "/dev/sdb1", O_RDONLY|O_DIRECT) = 4
mmap(NULL, 33558528, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f46451db000
read(4, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 33554432) = 33554432
write(1, "Time used: 1.447891 s to read 33"..., 45) = 45
close(4)                                = 0
munmap(0x7f46451db000, 33558528)        = 0
nanosleep({tv_sec=1, tv_nsec=0}, 0x7ffed794fdb0) = 0
openat(AT_FDCWD, "/dev/sdb1", O_RDONLY|O_DIRECT) = 4
mmap(NULL, 33558528, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f46451db000
read(4, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 33554432) = 33554432
write(1, "Time used: 1.448310 s to read 33"..., 45) = 45
close(4)                                = 0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# 删除上述案例应用
$ docker rm -f app

# 运行修复后的应用
$ docker run --privileged --name=app -itd feisky/app:io-cached

root@perf:/home/ykg# docker logs app
Reading data from disk /dev/sdb1 with buffer size 33554432
Time used: 1.396874 s to read 33554432 bytes
Time used: 0.018585 s to read 33554432 bytes
Time used: 0.017417 s to read 33554432 bytes
Time used: 0.016598 s to read 33554432 bytes
Time used: 0.017005 s to read 33554432 bytes
Time used: 0.016940 s to read 33554432 bytes
Time used: 0.016698 s to read 33554432 bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
22:08:15 Buffers MB: 46 / Cached MB: 772 / Sort: HITS / Order: ascending
PID      UID      CMD              HITS     MISSES   DIRTIES  READ_HIT%  WRITE_HIT%
    1121 root     atopacctd               1        0        0     100.0%       0.0%
    1504 root     python3                 2        0        0     100.0%       0.0%
    4362 root     dockerd                 4        0        1      75.0%       0.0%
    4404 root     dockerd                 4        0        1      75.0%       0.0%
    4433 root     dockerd                12        0        3      75.0%       0.0%
    4303 root     cachetop               15        0        0     100.0%       0.0%
     376 root     jbd2/sda1-8            17       13       11      20.0%       6.7%
    1442 root     python3                47        4        2      88.2%       3.9%
    5079 root     sh                    110        0        0     100.0%       0.0%
    5081 root     sh                    114        0        0     100.0%       0.0%
    5078 root     python3               366        0        0     100.0%       0.0%
    5080 root     python3               366        0        0     100.0%       0.0%
    5080 root     sh                    389        0        0     100.0%       0.0%
    5078 root     sh                    410        0        0     100.0%       0.0%
    5079 root     iptables              442        0        0     100.0%       0.0%
    5081 root     iptables              521        0        0     100.0%       0.0%
    4988 root     app                 40960        0        0     100.0%       0.0%