Let’s say you are on a system and a program stops working.
You check the space on disk, check that no one has modified the config files, check things like dns, etc… everything seems normal and you don’t know what else to check.
It could be that the filesystem got corrupted after a powerdown, for example, and one file or more are corrupted and this would be hard to figure out.
To find whats going wrong then you can use strace.
In the simplest case strace runs the specified command until it exits. It intercepts and records the system calls which are called by a process and the signals which are received by a process. The name of each system call, its arguments and its return value are printed on standard error or to the file specified with the -o option.
http://linux.die.net/man/1/strace
As you may know the programs request system calls, and get signals from the Operating System/Kernel.
strace will show all those requests done by the program, and the signals received. That means that you will see the requests from the program to the kernel to open a file, for example a config file.
Executing:
strace /usr/bin/ssh
That is the sample output:
strace /usr/bin/ssh
execve("/usr/bin/ssh", ["/usr/bin/ssh"], [/* 61 vars */]) = 0
brk(0) = 0x7fc71509c000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713cb2000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=126104, ...}) = 0
mmap(NULL, 126104, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc713c93000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=134224, ...}) = 0
mmap(NULL, 2234088, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc713870000
mprotect(0x7fc71388f000, 2097152, PROT_NONE) = 0
mmap(0x7fc713a8f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7fc713a8f000
mmap(0x7fc713a91000, 1768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc713a91000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\361\5\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1934816, ...}) = 0
mmap(NULL, 4045240, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc713494000
mprotect(0x7fc713646000, 2097152, PROT_NONE) = 0
mmap(0x7fc713846000, 155648, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b2000) = 0x7fc713846000
mmap(0x7fc71386c000, 14776, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc71386c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14664, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c92000
mmap(NULL, 2109736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc713290000
mprotect(0x7fc713293000, 2093056, PROT_NONE) = 0
mmap(0x7fc713492000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fc713492000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\36\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=100728, ...}) = 0
mmap(NULL, 2195784, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc713077000
mprotect(0x7fc71308f000, 2093056, PROT_NONE) = 0
mmap(0x7fc71328e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fc71328e000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320:\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=97144, ...}) = 0
mmap(NULL, 2202280, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc712e5d000
mprotect(0x7fc712e73000, 2097152, PROT_NONE) = 0
mmap(0x7fc713073000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7fc713073000
mmap(0x7fc713075000, 6824, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc713075000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\234\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=252704, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c91000
mmap(NULL, 2348608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc712c1f000
mprotect(0x7fc712c5a000, 2097152, PROT_NONE) = 0
mmap(0x7fc712e5a000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3b000) = 0x7fc712e5a000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\36\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1853400, ...}) = 0
mmap(NULL, 3961912, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc712857000
mprotect(0x7fc712a14000, 2097152, PROT_NONE) = 0
mmap(0x7fc712c14000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x7fc712c14000
mmap(0x7fc712c1a000, 17464, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc712c1a000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpcre.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\31\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=256224, ...}) = 0
mmap(NULL, 2351392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc712618000
mprotect(0x7fc712655000, 2097152, PROT_NONE) = 0
mmap(0x7fc712855000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3d000) = 0x7fc712855000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360l\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=135757, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c90000
mmap(NULL, 2212936, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7123fb000
mprotect(0x7fc712412000, 2097152, PROT_NONE) = 0
mmap(0x7fc712612000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fc712612000
mmap(0x7fc712614000, 13384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc712614000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libkrb5.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260p\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=848672, ...}) = 0
mmap(NULL, 2944608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc71212c000
mprotect(0x7fc7121f1000, 2093056, PROT_NONE) = 0
mmap(0x7fc7123f0000, 45056, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc4000) = 0x7fc7123f0000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360;\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=158136, ...}) = 0
mmap(NULL, 2257008, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc711f04000
mprotect(0x7fc711f2a000, 2093056, PROT_NONE) = 0
mmap(0x7fc712129000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7fc712129000
mmap(0x7fc71212b000, 112, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc71212b000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libcom_err.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\24\0\0\0\0\0\0"..., 832) = 832
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c8f000
fstat(3, {st_mode=S_IFREG|0644, st_size=14592, ...}) = 0
mmap(NULL, 2109896, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc711d00000
mprotect(0x7fc711d03000, 2093056, PROT_NONE) = 0
mmap(0x7fc711f02000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fc711f02000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@ \0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31160, ...}) = 0
mmap(NULL, 2126632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc711af8000
mprotect(0x7fc711aff000, 2093056, PROT_NONE) = 0
mmap(0x7fc711cfe000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fc711cfe000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\17\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14256, ...}) = 0
mmap(NULL, 2109456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7118f4000
mprotect(0x7fc7118f6000, 2097152, PROT_NONE) = 0
mmap(0x7fc711af6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fc711af6000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c8e000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c8d000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713c8b000
arch_prctl(ARCH_SET_FS, 0x7fc713c8b840) = 0
mprotect(0x7fc712c14000, 16384, PROT_READ) = 0
mprotect(0x7fc711af6000, 4096, PROT_READ) = 0
mprotect(0x7fc713492000, 4096, PROT_READ) = 0
mprotect(0x7fc711cfe000, 4096, PROT_READ) = 0
mprotect(0x7fc712612000, 4096, PROT_READ) = 0
mprotect(0x7fc711f02000, 4096, PROT_READ) = 0
mprotect(0x7fc712129000, 4096, PROT_READ) = 0
mprotect(0x7fc713073000, 4096, PROT_READ) = 0
mprotect(0x7fc7123f0000, 40960, PROT_READ) = 0
mprotect(0x7fc712855000, 4096, PROT_READ) = 0
mprotect(0x7fc712e5a000, 4096, PROT_READ) = 0
mprotect(0x7fc71328e000, 4096, PROT_READ) = 0
mprotect(0x7fc713846000, 110592, PROT_READ) = 0
mprotect(0x7fc713a8f000, 4096, PROT_READ) = 0
mprotect(0x7fc713f1f000, 8192, PROT_READ) = 0
mprotect(0x7fc713cb4000, 4096, PROT_READ) = 0
munmap(0x7fc713c93000, 126104) = 0
set_tid_address(0x7fc713c8bb10) = 13672
set_robust_list(0x7fc713c8bb20, 24) = 0
futex(0x7fff5c43f09c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7fc713c8b840) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7fc7124017e0, [], SA_RESTORER|SA_SIGINFO, 0x7fc71240abb0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fc712401860, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fc71240abb0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7fff5c43f090) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7fff5c43f090) = -1 ENOENT (No such file or directory)
brk(0) = 0x7fc71509c000
brk(0x7fc7150bd000) = 0x7fc7150bd000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713cb1000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 1024) = 328
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fc713cb1000, 4096) = 0
open("/dev/null", O_RDWR) = 3
close(3) = 0
openat(AT_FDCWD, "/proc/13672/fd", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 6 entries */, 32768) = 144
getdents(3, /* 0 entries */, 32768) = 0
close(3) = 0
getuid() = 1000
geteuid() = 1000
setresuid(-1, 1000, -1) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=513, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc713cb1000
read(3, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 513
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7fc713cb1000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=126104, ...}) = 0
mmap(NULL, 126104, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc713c93000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libnss_compat.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\23\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=35728, ...}) = 0
mmap(NULL, 2131288, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7116eb000
mprotect(0x7fc7116f3000, 2093056, PROT_NONE) = 0
mmap(0x7fc7118f2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7fc7118f2000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libnsl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`A\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=97296, ...}) = 0
mmap(NULL, 2202360, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7114d1000
mprotect(0x7fc7114e8000, 2093056, PROT_NONE) = 0
mmap(0x7fc7116e7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7fc7116e7000
mmap(0x7fc7116e9000, 6904, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc7116e9000
close(3) = 0
mprotect(0x7fc7116e7000, 4096, PROT_READ) = 0
mprotect(0x7fc7118f2000, 4096, PROT_READ) = 0
munmap(0x7fc713c93000, 126104) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=126104, ...}) = 0
mmap(NULL, 126104, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc713c93000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=47760, ...}) = 0
mmap(NULL, 2143616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7112c5000
mprotect(0x7fc7112d0000, 2093056, PROT_NONE) = 0
mmap(0x7fc7114cf000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7fc7114cf000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=52160, ...}) = 0
mmap(NULL, 2148504, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc7110b8000
mprotect(0x7fc7110c4000, 2093056, PROT_NONE) = 0
mmap(0x7fc7112c3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7fc7112c3000
close(3) = 0
mprotect(0x7fc7112c3000, 4096, PROT_READ) = 0
mprotect(0x7fc7114cf000, 4096, PROT_READ) = 0
munmap(0x7fc713c93000, 126104) = 0
open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
lseek(3, 0, SEEK_CUR) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=1823, ...}) = 0
mmap(NULL, 1823, PROT_READ, MAP_SHARED, 3, 0) = 0x7fc713cb1000
lseek(3, 1823, SEEK_SET) = 1823
munmap(0x7fc713cb1000, 1823) = 0
close(3) = 0
umask(022) = 022
write(2, "usage: ssh [-1246AaCfgKkMNnqsTtV"..., 466usage: ssh [-1246AaCfgKkMNnqsTtVvXxYy] [-b bind_address] [-c cipher_spec]
[-D [bind_address:]port] [-e escape_char] [-F configfile]
[-I pkcs11] [-i identity_file]
[-L [bind_address:]port:host:hostport]
[-l login_name] [-m mac_spec] [-O ctl_cmd] [-o option] [-p port]
[-R [bind_address:]port:host:hostport] [-S ctl_path]
[-W host:port] [-w local_tun[:remote_tun]]
[user@]hostname [command]
) = 466
exit_group(255) = ?
+++ exited with 255 +++
You can also generate a log with that info:
strace -o test_log.txt /usr/bin/ssh
Let’s pay attention to the open files:
Here we can see what files were open, the mode and the result.
So if your program failen opening a certain file you will see it on the traces.
Also we can review the access:
cat test_log.txt | grep access --after-context=2
You can specify to trace only certain set of system calls by passing parameter -e trace=open,close,read,write,stat,chmod,unlink or -e trace=network or -e trace=process or -e trace=memory or -e trace=ipc or -e trace=signal etcetera.
Can also dump data read -e read=set or -e write=set for a full hexadecimal and ASCII dump of all the data written to file descriptors listed in the specified set… or -e signal=set (default signal=all) or even by negation -e signal =! SIGIO (or signal=!io)…
You can also trace libraries with ltrace or processes with ptrace.
But strace is very handy, as you can attach to it and detach and the process will continue working.
To attach to a process use -p and see the flow continuously with -f
strace -fp 2418 -s 255
And see the open files with lsof.
You can use lsof to see the TCP connections, as socket are also kind of files:
lsof -iTCP:80
You can also know information of what process is owner of a tcp/udp connection with ss or the older netstat:
netstat -tnp
Take a look at ss for advanced sockets inspecting. The most typical call, which is also easy to remember, is:
ss -plant
Of course you will find very interesting info on /proc pseudo-filesystem.
You can troubleshoot the environment for the process by doing:
strings /proc/1714/environ
Where 1714 is the process id, whatever.
/proc/[pid]/fd/ is a subdirectory containing one entry for each file open by the process, named by its file descriptor, being a symbolic link to the actual file.
/proc/[pid]/fdinfo/ will show information on the flags for the access mode of the open files and /proc/[pid]/io contains input/outputs statistics for the process.
My Open Source monitoring project ctop.py gets most of the information from /proc