From 167f968660e1091c8f9c288ac909731cc5431e62 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 22 Oct 2021 11:32:17 +0200 Subject: [PATCH 1/5] Add layers to image data Reference: https://github.com/nexB/scancode.io/issues/175 Reportedby: Sameer1046 @Sameer1046 Signed-off-by: Philippe Ombredanne --- scanpipe/pipes/docker.py | 2 +- scanpipe/tests/data/docker-images.tar.gz | Bin 0 -> 6773 bytes .../docker-images.tar.gz-expected-data-1.json | 394 ++++++++++++++++++ 3 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 scanpipe/tests/data/docker-images.tar.gz create mode 100644 scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index 74af0389cd..e510e72679 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -75,7 +75,7 @@ def get_image_data(image): """ Returns a mapping of image-related data given an `image`. """ - exclude = ["extracted_location", "archive_location", "layers"] + exclude = ["extracted_location", "archive_location"] image_data = { key: value for key, value in image.to_dict().items() if key not in exclude } diff --git a/scanpipe/tests/data/docker-images.tar.gz b/scanpipe/tests/data/docker-images.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..14169707d03e3e5d5a62996caf91ddc83fcafd98 GIT binary patch literal 6773 zcmZu#Ran%4v!xpW>5}fw1xb;XMkN+lQo2E!mF`YySdcF1E-C2{L{eJ7C6=U z?mV1%nVE0Sd}khJ7~*hnW~IU*DCqXi7PdB?{9YCwC@0d6uC$=myFlo$JYj=oE;ZuG zjmiAVsY!FY|D%4h8nK&dVy)s5jtES2!|Bt53cRqZZ}&x_HTdg`;Ng{xTAUS{l&F6u zEg&0N8YxwZqwO4 zD&gzfB9Aa#E0D<0{LJ27nuFv5<*?trg||Pl8*~j*lD?|%Nx3?Z?3s%if!hhw$+~!H zt*z{&k$Q@@Iiv%2CmKV~aQhz^WwZ55jc4u^K@R+E_!LZ?!d#qMD8h z89*MsP6XJn@zIfkdJxhl6Ugd$3gZZv0kybiJhCVXOvsJe@csd~eP>WZuh3To4b-=VDHMGR8Lm7_~#&FazzhraxTm!g6*5gPP z^|lK@Y`#2_>*T!XqBVzXpizuNtDzExJn)%;h?zA%gF3)9zFXR6gGJW376$Bm;5T8`G$>_wX$&};kas``tEd0G+O^gc8a@29GFzXY*FQP*BT}4@L*tk++4I_ zbnY0}&@p#uW&EM(r0)Ql?UA=TtZNDnA@OVh3k?3t~T+>2(F5(DpuNv8dPfs^8DEPcC z-|;_874(v_(tqpAL9pfSExZ=)gX67ildYIwtNjf0iUK)QlqWvjF~RAXTQm`VeIOQvXr=O=D;Q1{%mbn`}rjhacgrKSdP0 zdm%U#)*BOiWxq(16cTXtNfCQPEB2K)O%LaVPtgmI759h0x43q(%E>F9jAPzfBs}!m zO5X|hS+97TeyQ@OGl_cgdS*6=dMdt2D>GzF&c+&A^Ct|_uI2;0l1IfK}I78%@O~cwN5H+--7d5PhzeWk6 z!gX4kOFWth>s8seCXzoa$2Z6!Y@^pObvVr_OJ4zgB6xAh(;g%hBidY%;#N*k5(v@rP9Y2y_R`6+hUWIJ~@0znyRSXsS z!1;4?Usy`+>ZKgR;{^eZ>O1wywx1~){WU%tYH_MLB!5LBsi%cMXyiPPlU67EAU-r` z_(^C`$LUtGh3V;R=x)a+i9h&m{^!IKwe?&z=ZhbvFIhOe_NRzQvUBW{`WqA*3m@GE zHYIq73gD=H@|%hHkjxF#NRb{8^`4P<1G-53F)DYOAj?L^fD63+F=o&#P85;)SbpyH??UTH$BZ z5^%=E^rdv+LBlsK_o+7$89A5t3R`v0zt(m=EfJV(yOeK6d;rU1I_tBAqk$LUb}PqY z6sbKfN!L=oSatXLTKEgW^+{b7ul;Gf>W)J8QY!?&ox@<6Nz97h_D2n}N7oNQOZxO{ zdf|AHUX%{1zu-8LQuOf;HL&S^@;_~yMTgA+>xp(Z*>;ZHI#F|4^+Uec6BO2Y?q62l z1u0ZIwE3s9tijg#UA;?>h^g;avrLsFch;$W<6fzCEr)%Al~Ylqdv(@~eo`E7wp%dE zRX`8T6M8;Cl_IXcE}r#Q{0N1-Qsy8C z`l%IO*j#F!Bj0WOH3M^2;gnRiw|ia=p7f(cWDME}fpq@i?Gx4Il3}qm0o+rsS3m&o zRluEtK}cg9zN_3K=Bw3ypILgKlj3OL79AY*pcOr;va@Tg!?vNhG%xWZ_1Wdr*M5`N z`7=DOXDIc>4Fb)ZGR;0``uJXtsE5a;_l9g=nyY^q?dBjT1{eDO1nMPlylakTH6H0e zv-&+MzsJ4mv9!5NRQ&k-C}YGX!rsWrNW5{rw(cvQ_{wMg1$mqf6T|AI+w~#Wq(Jii zlC~Mh$*H$LHg>=Rw(CuSTo97ef5k-KRgjxU#N@XJrquQ$O`3ZLi`v&bQCq+~YM+Uda_PFM_loY^E5~)=EM#>5EEJgHXfy3c6P7@S(1m6=&Iy4<@vl#m38yL8=J+{c_H!6<*_Q1a zS^%p>T=1|Ln9)|wD|oI`exd=LM^gU zb3*M| z-1>#WdtslCn!TA{Z2bI|ba~}Xj2T?!Y0pT&|J~`th6!Sc5U-dME7vn03%b4vbUawg z<-pM)2>}+>e?IJ=jO1=tzXU$~wjA0~PC0n@{#4AMe5c{Y06ois!716s6}yh$VWmmN z`)Gil{Bh$D0M=FNS=*gwJ4gkP69Yh4{nNml|)Qo>Xb6HWQDhq9vxZ>QJMm zTVA|-!Hx$0(Y`-rJ0J1z3rnNjY|I<{Z@9O=%igQ2vUR_Ed4iF;$n}h3O()&oDuQ|A zXe4(Aa(?k$d|gs^Pz-CCv*LYr4aS28(};zyNHQqjJ((w2re0!sx}PpWl4x+>oLldc z3IvYiDAChN#0f@$DwOyfbjtrQyQp_(kfyT!ZrnmZg|st!bHVhjaslH zbYwJ}=|@@~Fn-`5?&ZldnjWG@!=}hFR?^MHMP}WQTQqe6^&ei{EyT}so{2!L#Auy0|5Gl5O7zj!_9d(m z;+3^enX8b&psNqaT^2=yap{3^FCp7&@+Da4#9Ghom2by)UN4OPC5f8)QPx*YVOB%4 zIwlJHF_K_2^x(R@tt+%iy|Uqjn1D29fAQsyi?s9uH3=h1NDGuA5=FQm2>o)W;%v`c z;*;Hs%7C4y;w^f~tX*$cN+q+%3xThU zXV&CbA|ytyF1Qc>7)blDx`?q*Y6GV1*6$LCTI2BxU-6B+Vp~jnQZG_k^kljXK;w_*f#xy_#T_aEfborJ%^p~r~ui6{(GjZ`-~-%dqz$p=3pP>6(>G@ zZuStwkyjmV5zcwAFjNXjJ##}?3qR|0KmVepy6_b?*e(|FTW@t}7|xF#$rGVO@i$O# zSS#qx8M_J|*!E$=A{mdaG&Dw0zJi-Pe?z}LYOX3ts!${HhdCt|yZ(%OY{!Kn%eK%* z#x3fF@8;!n)>1P~i={ZEeEL`x?Ur0D3aZL~l#MzM)6~i4@;*5aLprsuZ&`ijZWv(0 zItSdbUJAIVD_;)d^t~Qvc_tL`sv;;bg3j;y+@@?gi#??9T1HINOHMNqelt%CJZQVV zVp74iF(#Glsq!zxFhhKqwQ2e;zcElVD6vT!v9PK{`#7}Eed^Ysa8{R7y*8j}W{sFR zdCjjc3k}&9z7@Wli~xIket@C3(UAJaPkrudM>F>ivCr^n!}IkAS6rWD#!=F5vbXol z{RmO>)GumdRV5&E)+6g4QFoq5-Zdk}I83@cxMyQq{>o>R zNSfYtVspSqbgn*OnnITvC6YIUAgD&^cO1MRPc1okS?Up=&aU0DCV~2#$~K0($Shj* zISKcCx9pg{`=XxGlqwD{d+|l=J4K>-k`j?0E#^I0>%l`dYIJ(BYMtHiX`jQx*PI8+ z)?{=h2XPB@WVZY^7x)y>t;*ynrPc`TpAx7grG%3rD5({=!n&Uo=7qUcVs0k&!OvSc z`Srhw>YsFzWtrh}*>f8`q8F@Eab4VIO7F-kjy+Yea$@q-z7t3ES77Y0qTN*H0Tv~f zCZiS5YMj z@e=q;Sh?24SnauZg`$a|nv(wY5gWc==u}fOVMxY)seX-V z770e#0Y(xW9!_PD^>Kji+oGqoewRmv;oClv;*X8u#K2B06W|zfbj=V6?BSw}0u+Xb zz84a!S9j2?ZnsF~AyT`vECNNUFEZ_h+y46TnfRpP#or_BLO*Iy@5eXm+LMxA^hG|_ z|3YHVLzSOK#pRJ8m$$3X%|`)JIlWuXUr^GMGdZ0xKpDdo?JRX+maAHCyg>s&R@1`^ zmfvtMX)3GU?MX7uXe#FC8u{#w#(JK2GY-T)9aQ!q03mtgGTM*7a>)SEEU=8>h%{Iw z6u)Eee2-D`*=-}GNRkz%sCUPWuO*BsDpAkKTEet~gBpIeC`{4LH^4at&|LI6Jp5xO z6pHL)|KH9|k#Z|{0LJSF*AP>nP!S1YUSvO}X9qwmcy|!ew{1|@M^`BzcksLqm;VkD zG!5rLQc6<2C+lj*#9lK}F2%NJChDjS^Q7=~8)S!}bHrv^B`8qJP*%o`;9vA*KGrWG zvBv=%GC=@L$_UVcs{Z!*0FX8SMl}s+HXH-5ks$U(EJs3e0GA3_!T85c=aFL{a0z>Z zG+=1*qX&#m-zXxN zK2n>r-gxCbBE9l6C0B7;dx9$rJ#q7l2OaV?A!wY9BVz zz*5%b1RxE4fND1ELtEBSB*CwzmpK0Eh7n*;K`vvA{hw+`UjqMrF0BJDj~xHlkT`_3 zME-`pRgZbB$E;d~X5VTv2>}MAHFppqpwu$+Sd8(5oeVa+bOcU{ zU3AZp9N7al-uitZGvF9{bX|V{74}KYzZFRQ*SXs`z}+>#4jjLFW2v7Wn-Y`*%)M4V zZ5(|366vAY(wkho_BQm{E0Oxpv2Hy>?(8^eKdX%7U&5*(<;jrAvi|t5C5Edsth2cG&&HY|D zEs#cT&spf+y_9*l8s69)-WaGxd`^0pfZeV1LdxLqzi$={CRT@Ge6RvHZJ7Er2R66f zpY*I8i6T8I3QvY9Q~pxMw08yiZ{l_5+4wEZnq55pE(c%_)HxxwNmkdl*C-jBNKX&; zo)9fJ{tp*NuD=iKZ`8{>j+zU5b4G{%>{hRgkDqw$C%ycucUIRW{q*PJf&&3Lxo~<3 z={}L|d#h7$u;Kp0HfzDCvi47e&Z%0bAwdNsp-nTjn>!9Ui%3UVy2Ua2A#?lW>vGx_mA1U21*=$V}* zK5zAIbX{6C^)dTno@L>5;p~<6U~4<76fPm>alw0!7nv3umoV>8%!5NHsDPRJ9Y zAKvSH!O`AXd7SMvp)iIDBjWDnS0|~=UN|A$PbEaw_p03P>Lt6ZeVl0=%k0go-H$Ued3F`= zQwtRD2KEU7FytHB?%;=8vyThQuLFrt(OhNyAB@e9)mqYaHgZ7{4ot*27-%8#jQ_uj QCfO389A{2Mp+ZIZFL!^2cK`qY literal 0 HcmV?d00001 diff --git a/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json b/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json new file mode 100644 index 0000000000..b2e5c9f37f --- /dev/null +++ b/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json @@ -0,0 +1,394 @@ +[ + { + "sha256": null, + "docker_version": "1.12.6", + "os": "linux", + "os_version": null, + "architecture": "amd64", + "variant": null, + "created": "2017-01-18T09:49:05.022760406Z", + "author": null, + "comment": null, + "labels": { + "com.microscaling.docker.dockerfile": "/Dockerfile", + "com.microscaling.license": "Apache-2.0", + "org.label-schema.build-date": "2017-01-18T09:49:02Z", + "org.label-schema.description": "Our Microscaling Engine provides automation, resilience and efficiency for microservice architectures. Experiment with microscaling at app.microscaling.com.", + "org.label-schema.name": "Microscaling Engine", + "org.label-schema.schema-version": "1.0", + "org.label-schema.url": "https://microscaling.com", + "org.label-schema.vcs-ref": "45b22cb", + "org.label-schema.vcs-url": "https://github.com/microscaling/microscaling.git", + "org.label-schema.vendor": "Microscaling Systems", + "org.label-schema.version": "0.9.1" + }, + "image_format": "docker", + "image_id": "4ac5c955a6dc9dc709fa2ce8dd21c45292afd9e4af38dcd1fb239ed546d19233", + "config_digest": "sha256:4ac5c955a6dc9dc709fa2ce8dd21c45292afd9e4af38dcd1fb239ed546d19233", + "tags": [ + "microscaling/microscaling:latest" + ], + "distro": null, + "layers": [ + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/d388bee71bbf28f77042d89b353bacd14506227a39b6b1098afe457f0a0b608e/layer.tar", + "sha256": "7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2016-12-27T18:17:13.762716133Z", + "created_by": "/bin/sh -c #(nop) ADD file:eeed5f514a35d18fcd9cbfe6c40c582211020bffdd53e4799018d33826fe5067 in / ", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/e439273044a3b34ba206c3598cd007dbba34db5af98d40718d585869af68602e/layer.tar", + "sha256": "d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-03T13:15:58.410035553Z", + "created_by": "/bin/sh -c apk update && apk upgrade && apk add $BUILD_PACKAGES && rm -rf /var/cache/apk/*", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/ca473f7411dd74e2d4781fc552d75d3b63c830ff66a253eec6bf5a6261c031b2/layer.tar", + "sha256": "d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-17T11:17:46.675078318Z", + "created_by": "/bin/sh -c #(nop) COPY multi:0847be51b158e2902e11d406f498a1d8d58ede5fbab01297b446d84b42667433 in / ", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/325919382e7b94a21ef80452c2fb599f5a03364e8559da0eca1359ad6ac394a0/layer.tar", + "sha256": "76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-17T11:17:48.829523581Z", + "created_by": "/bin/sh -c chmod +x /microscaling", + "comment": null + } + ], + "history": [ + { + "created": "2016-12-27T18:17:13.762716133Z", + "created_by": "/bin/sh -c #(nop) ADD file:eeed5f514a35d18fcd9cbfe6c40c582211020bffdd53e4799018d33826fe5067 in / " + }, + { + "created": "2017-01-03T13:15:49.512603795Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) MAINTAINER Ross Fairbanks \"ross@microscaling.com\"", + "empty_layer": true + }, + { + "created": "2017-01-03T13:15:49.913789767Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ENV BUILD_PACKAGES=ca-certificates", + "empty_layer": true + }, + { + "created": "2017-01-03T13:15:58.410035553Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c apk update && apk upgrade && apk add $BUILD_PACKAGES && rm -rf /var/cache/apk/*" + }, + { + "created": "2017-01-17T11:17:46.675078318Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) COPY multi:0847be51b158e2902e11d406f498a1d8d58ede5fbab01297b446d84b42667433 in / " + }, + { + "created": "2017-01-17T11:17:48.829523581Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c chmod +x /microscaling" + }, + { + "created": "2017-01-17T11:17:49.322379301Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG BUILD_DATE", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:49.639696203Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VERSION", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:49.991577291Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VCS_URL", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:50.34992704Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VCS_REF", + "empty_layer": true + }, + { + "created": "2017-01-18T09:49:04.672769694Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) LABEL org.label-schema.build-date=2017-01-18T09:49:02Z org.label-schema.name=Microscaling Engine org.label-schema.description=Our Microscaling Engine provides automation, resilience and efficiency for microservice architectures. Experiment with microscaling at app.microscaling.com. org.label-schema.url=https://microscaling.com org.label-schema.vcs-url=https://github.com/microscaling/microscaling.git org.label-schema.vcs-ref=45b22cb org.label-schema.vendor=Microscaling Systems org.label-schema.version=0.9.1 org.label-schema.schema-version=1.0 com.microscaling.docker.dockerfile=/Dockerfile com.microscaling.license=Apache-2.0", + "empty_layer": true + }, + { + "created": "2017-01-18T09:49:05.022760406Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) CMD [\"/microscaling\"]", + "empty_layer": true + } + ] + }, + { + "sha256": null, + "docker_version": "19.03.8", + "os": "linux", + "os_version": null, + "architecture": "amd64", + "variant": null, + "created": "2020-10-11T21:21:00.914818853Z", + "author": null, + "comment": null, + "labels": { + "maintainer": "Bjoern Kimminich ", + "org.opencontainers.image.authors": "Bjoern Kimminich ", + "org.opencontainers.image.created": "\u201d2020-10-11T21:19:43Z\u201d", + "org.opencontainers.image.description": "Capture-the-Flag (CTF) environment setup tools for OWASP Juice Shop", + "org.opencontainers.image.documentation": "https://help.owasp-juice.shop/part1/ctf.html", + "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.revision": "55df34d", + "org.opencontainers.image.source": "https://github.com/bkimminich/juice-shop-ctf.git", + "org.opencontainers.image.title": "OWASP Juice Shop CTF-Extension", + "org.opencontainers.image.url": "https://owasp-juice.shop", + "org.opencontainers.image.vendor": "Open Web Application Security Project", + "org.opencontainers.image.version": "8.1.2" + }, + "image_format": "docker", + "image_id": "2e92ac90bc1e459ee9d2d72a8bcd3787895d3d7d2d75391e620cea6f20b61315", + "config_digest": "sha256:2e92ac90bc1e459ee9d2d72a8bcd3787895d3d7d2d75391e620cea6f20b61315", + "tags": [ + "bkimminich/juice-shop-ctf:latest" + ], + "distro": null, + "layers": [ + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/2ba5d0825bcb0d843f975f7ba52db62531dbc73df083fae3d355c1a22d4a4bdc/layer.tar", + "sha256": "3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-04-24T01:05:03.608058404Z", + "created_by": "/bin/sh -c #(nop) ADD file:b91adb67b670d3a6ff9463e48b7def903ed516be66fc4282d22c53e41512be49 in / ", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/c9d345514b7a7f4e4d14748c6a7a6cb408f117fccce87ef80bf72429b5360c44/layer.tar", + "sha256": "09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:14.227103847Z", + "created_by": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps curl && ARCH= && alpineArch=\"$(apk --print-arch)\" && case \"${alpineArch##*-}\" in x86_64) ARCH='x64' CHECKSUM=\"b8b33ec1d2f6463ded7c02bf29ef4141ff4aabb2c0e0ba14cf16ac7c0a6a31a5\" ;; *) ;; esac && if [ -n \"${CHECKSUM}\" ]; then set -eu; curl -fsSLO --compressed \"https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\"; echo \"$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" | sha256sum -c - && tar -xJf \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" -C /usr/local --strip-components=1 --no-same-owner && ln -s /usr/local/bin/node /usr/local/bin/nodejs; else echo \"Building from source\" && apk add --no-cache --virtual .build-deps-full binutils-gold g++ gcc gnupg libgcc linux-headers make python3 && for key in 4ED778F539E3634C779C87C6D7062848A1AB005C 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 1C050899334244A8AF75E53792EF661D867B9DFA 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C DD8F2338BAE7501E3DD5AC78C273792F7D83545D A48C2BEE680E841632CD4E44F07496B3EB3C1762 108F52B48DB57BB0CC439B2997B01419BD92F80A B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps-full && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt; fi && rm -f \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" && apk del .build-deps && node --version && npm --version", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/cbfc5b91e94f389bd405928c3be3367feea459d5917bde9d8b2792b96ae789ef/layer.tar", + "sha256": "55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:16.985023204Z", + "created_by": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn && yarn --version", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/bf1ad815335aca28dc1b64e4a9cad7f0bd9f7e9ee8162dce166150b4f1169b0e/layer.tar", + "sha256": "01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:17.192305843Z", + "created_by": "/bin/sh -c #(nop) COPY file:238737301d47304174e4d24f4def935b29b3069c03c72ae8de97d94624382fce in /usr/local/bin/ ", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/5b2fc24305d353053e3a6aa7a613e75240e34cfede725f4bd4c466468cd3c48c/layer.tar", + "sha256": "08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-11T21:20:59.851868447Z", + "created_by": "/bin/sh -c #(nop) COPY dir:5bd634490f1e144602e7b39a1ebcb69af1748a34673299424a7fe6f10334bfa1 in /juice-shop-ctf ", + "comment": null + }, + { + "extracted_location": null, + "archive_location": "/tmp/tmpnnbxlxvk/ee512e2eb28e302e80a1e0bfad85ee3b18adc84a80e88ceac4325c13e3e083ec/layer.tar", + "sha256": "5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-11T21:21:00.668316194Z", + "created_by": "/bin/sh -c #(nop) WORKDIR /data", + "comment": null + } + ], + "history": [ + { + "created": "2020-04-24T01:05:03.608058404Z", + "created_by": "/bin/sh -c #(nop) ADD file:b91adb67b670d3a6ff9463e48b7def903ed516be66fc4282d22c53e41512be49 in / " + }, + { + "created": "2020-04-24T01:05:03.92860976Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\"]", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:07.591276187Z", + "created_by": "/bin/sh -c #(nop) ENV NODE_VERSION=14.13.1", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:14.227103847Z", + "created_by": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps curl && ARCH= && alpineArch=\"$(apk --print-arch)\" && case \"${alpineArch##*-}\" in x86_64) ARCH='x64' CHECKSUM=\"b8b33ec1d2f6463ded7c02bf29ef4141ff4aabb2c0e0ba14cf16ac7c0a6a31a5\" ;; *) ;; esac && if [ -n \"${CHECKSUM}\" ]; then set -eu; curl -fsSLO --compressed \"https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\"; echo \"$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" | sha256sum -c - && tar -xJf \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" -C /usr/local --strip-components=1 --no-same-owner && ln -s /usr/local/bin/node /usr/local/bin/nodejs; else echo \"Building from source\" && apk add --no-cache --virtual .build-deps-full binutils-gold g++ gcc gnupg libgcc linux-headers make python3 && for key in 4ED778F539E3634C779C87C6D7062848A1AB005C 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 1C050899334244A8AF75E53792EF661D867B9DFA 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C DD8F2338BAE7501E3DD5AC78C273792F7D83545D A48C2BEE680E841632CD4E44F07496B3EB3C1762 108F52B48DB57BB0CC439B2997B01419BD92F80A B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps-full && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt; fi && rm -f \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" && apk del .build-deps && node --version && npm --version" + }, + { + "created": "2020-10-08T16:23:14.624783745Z", + "created_by": "/bin/sh -c #(nop) ENV YARN_VERSION=1.22.5", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:16.985023204Z", + "created_by": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn && yarn --version" + }, + { + "created": "2020-10-08T16:23:17.192305843Z", + "created_by": "/bin/sh -c #(nop) COPY file:238737301d47304174e4d24f4def935b29b3069c03c72ae8de97d94624382fce in /usr/local/bin/ " + }, + { + "created": "2020-10-08T16:23:17.37961953Z", + "created_by": "/bin/sh -c #(nop) ENTRYPOINT [\"docker-entrypoint.sh\"]", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:17.549978283Z", + "created_by": "/bin/sh -c #(nop) CMD [\"node\"]", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:52.959207202Z", + "created_by": "/bin/sh -c #(nop) ARG BUILD_DATE", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:53.20642118Z", + "created_by": "/bin/sh -c #(nop) ARG VCS_REF", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:53.439387368Z", + "created_by": "/bin/sh -c #(nop) LABEL maintainer=Bjoern Kimminich org.opencontainers.image.title=OWASP Juice Shop CTF-Extension org.opencontainers.image.description=Capture-the-Flag (CTF) environment setup tools for OWASP Juice Shop org.opencontainers.image.authors=Bjoern Kimminich org.opencontainers.image.vendor=Open Web Application Security Project org.opencontainers.image.documentation=https://help.owasp-juice.shop/part1/ctf.html org.opencontainers.image.licenses=MIT org.opencontainers.image.version=8.1.2 org.opencontainers.image.url=https://owasp-juice.shop org.opencontainers.image.source=https://github.com/bkimminich/juice-shop-ctf.git org.opencontainers.image.revision=55df34d org.opencontainers.image.created=\u201d2020-10-11T21:19:43Z\u201d", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:59.851868447Z", + "created_by": "/bin/sh -c #(nop) COPY dir:5bd634490f1e144602e7b39a1ebcb69af1748a34673299424a7fe6f10334bfa1 in /juice-shop-ctf " + }, + { + "created": "2020-10-11T21:21:00.397239764Z", + "created_by": "/bin/sh -c #(nop) VOLUME [/data]", + "empty_layer": true + }, + { + "created": "2020-10-11T21:21:00.668316194Z", + "created_by": "/bin/sh -c #(nop) WORKDIR /data" + }, + { + "created": "2020-10-11T21:21:00.914818853Z", + "created_by": "/bin/sh -c #(nop) ENTRYPOINT [\"npx\" \"/juice-shop-ctf/bin/juice-shop-ctf.js\"]", + "empty_layer": true + } + ] + } +] \ No newline at end of file From d6e3c7a2062493189fe7059921f87ab0e7bf4f65 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 26 Oct 2021 10:56:45 +0200 Subject: [PATCH 2/5] Report relative path for layers We keep two paths segments in layers to provide better ways to further post-process image scans Reference: https://github.com/nexB/scancode.io/issues/175 Reported-by: Sameer1046 @Sameer1046 Signed-off-by: Philippe Ombredanne --- scanpipe/pipes/docker.py | 73 +++- .../docker-images.tar.gz-expected-data-1.json | 20 +- .../docker-images.tar.gz-expected-data-2.json | 394 ++++++++++++++++++ scanpipe/tests/test_pipes_docker.py | 93 +++++ 4 files changed, 557 insertions(+), 23 deletions(-) create mode 100644 scanpipe/tests/data/docker-images.tar.gz-expected-data-2.json create mode 100644 scanpipe/tests/test_pipes_docker.py diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index e510e72679..0d4700b0de 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -36,9 +36,11 @@ def extract_images_from_inputs(project): """ - Collects all the tarballs from the `project` input/ work directory, extracts each - tarball to the tmp/ work directory and collects the images. - Returns the `images` and `errors` that may have happen during the extraction. + Collects all the tarballs from the `project` input/ work directory, extracts + each tarball to the tmp/ work directory and collects the images. + + Returns the `images` and an `errors` list of error messages that may have + happen during the extraction. """ target_path = project.tmp_path images = [] @@ -46,38 +48,83 @@ def extract_images_from_inputs(project): for input_tarball in project.inputs(pattern="*.tar*"): extract_target = target_path / f"{input_tarball.name}-extract" - extract_errors = extract_archive(input_tarball, extract_target) - images.extend(Image.get_images_from_dir(extract_target)) - errors.extend(extract_errors) + imgs, errs = extract_image_from_tarball(input_tarball, extract_target) + images.extend(imgs) + errors.extend(errs) return images, errors +def extract_image_from_tarball(input_tarball, extract_target, verify=True): + """ + Extract images from an ``input_tarball`` to an ``extract_target`` directory + Path object and collects the extracted images. + + Returns the `images` and an `errors` list of error messages that may have + happen during the extraction. + """ + errors = list(extract_archive(location=input_tarball, target=extract_target)) + images = Image.get_images_from_dir( + extracted_location=str(extract_target), + verify=verify, + ) + return images, errors + + def extract_layers_from_images(project, images): """ - Extracts all layers from the provided `images` into the `project` codebase/ work + Extracts all layers from the provided `images` into the `project` codebase + work directory. + + Returns an `errors` list of error messages that may occur during the + extraction. + """ + + return extract_layers_from_images_to_base_path( + base_path=project.codebase_path, + images=images, + ) + + +def extract_layers_from_images_to_base_path( + base_path, + images, +): + """ + Extracts all layers from the provided `images` into the `base_path` work directory. - Returns the `errors` that may happen during the extraction. + + Returns an `errors` list of error messages that may occur during the + extraction. """ errors = [] + base_path = Path(base_path) for image in images: image_dirname = Path(image.extracted_location).name - target_path = project.codebase_path / image_dirname + target_path = base_path / image_dirname for layer in image.layers: + # TODO: using the layer may be less useful than using the layer sha256 extract_target = target_path / layer.layer_id - extract_errors = extract_archive(layer.archive_location, extract_target) + extract_errors = extract_archive( + location=layer.archive_location, + target=extract_target, + ) errors.extend(extract_errors) layer.extracted_location = str(extract_target) -def get_image_data(image): +def get_image_data(image, layer_path_segments=2): """ Returns a mapping of image-related data given an `image`. + Keep only ``layer_path_segments`` trailing layer location segments (or keep + the locations unmodified if ``layer_path_segments`` is 0). """ - exclude = ["extracted_location", "archive_location"] + exclude_from_img = ["extracted_location", "archive_location"] image_data = { - key: value for key, value in image.to_dict().items() if key not in exclude + key: value + for key, value in image.to_dict(layer_path_segments=layer_path_segments).items() + if key not in exclude_from_img } return image_data diff --git a/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json b/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json index b2e5c9f37f..a3315e39cd 100644 --- a/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json +++ b/scanpipe/tests/data/docker-images.tar.gz-expected-data-1.json @@ -32,7 +32,7 @@ "layers": [ { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/d388bee71bbf28f77042d89b353bacd14506227a39b6b1098afe457f0a0b608e/layer.tar", + "archive_location": "d388bee71bbf28f77042d89b353bacd14506227a39b6b1098afe457f0a0b608e/layer.tar", "sha256": "7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", "docker_version": null, "os": null, @@ -50,7 +50,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/e439273044a3b34ba206c3598cd007dbba34db5af98d40718d585869af68602e/layer.tar", + "archive_location": "e439273044a3b34ba206c3598cd007dbba34db5af98d40718d585869af68602e/layer.tar", "sha256": "d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", "docker_version": null, "os": null, @@ -68,7 +68,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/ca473f7411dd74e2d4781fc552d75d3b63c830ff66a253eec6bf5a6261c031b2/layer.tar", + "archive_location": "ca473f7411dd74e2d4781fc552d75d3b63c830ff66a253eec6bf5a6261c031b2/layer.tar", "sha256": "d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", "docker_version": null, "os": null, @@ -86,7 +86,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/325919382e7b94a21ef80452c2fb599f5a03364e8559da0eca1359ad6ac394a0/layer.tar", + "archive_location": "325919382e7b94a21ef80452c2fb599f5a03364e8559da0eca1359ad6ac394a0/layer.tar", "sha256": "76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", "docker_version": null, "os": null, @@ -207,7 +207,7 @@ "layers": [ { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/2ba5d0825bcb0d843f975f7ba52db62531dbc73df083fae3d355c1a22d4a4bdc/layer.tar", + "archive_location": "2ba5d0825bcb0d843f975f7ba52db62531dbc73df083fae3d355c1a22d4a4bdc/layer.tar", "sha256": "3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", "docker_version": null, "os": null, @@ -225,7 +225,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/c9d345514b7a7f4e4d14748c6a7a6cb408f117fccce87ef80bf72429b5360c44/layer.tar", + "archive_location": "c9d345514b7a7f4e4d14748c6a7a6cb408f117fccce87ef80bf72429b5360c44/layer.tar", "sha256": "09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", "docker_version": null, "os": null, @@ -243,7 +243,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/cbfc5b91e94f389bd405928c3be3367feea459d5917bde9d8b2792b96ae789ef/layer.tar", + "archive_location": "cbfc5b91e94f389bd405928c3be3367feea459d5917bde9d8b2792b96ae789ef/layer.tar", "sha256": "55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", "docker_version": null, "os": null, @@ -261,7 +261,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/bf1ad815335aca28dc1b64e4a9cad7f0bd9f7e9ee8162dce166150b4f1169b0e/layer.tar", + "archive_location": "bf1ad815335aca28dc1b64e4a9cad7f0bd9f7e9ee8162dce166150b4f1169b0e/layer.tar", "sha256": "01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", "docker_version": null, "os": null, @@ -279,7 +279,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/5b2fc24305d353053e3a6aa7a613e75240e34cfede725f4bd4c466468cd3c48c/layer.tar", + "archive_location": "5b2fc24305d353053e3a6aa7a613e75240e34cfede725f4bd4c466468cd3c48c/layer.tar", "sha256": "08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", "docker_version": null, "os": null, @@ -297,7 +297,7 @@ }, { "extracted_location": null, - "archive_location": "/tmp/tmpnnbxlxvk/ee512e2eb28e302e80a1e0bfad85ee3b18adc84a80e88ceac4325c13e3e083ec/layer.tar", + "archive_location": "ee512e2eb28e302e80a1e0bfad85ee3b18adc84a80e88ceac4325c13e3e083ec/layer.tar", "sha256": "5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", "docker_version": null, "os": null, diff --git a/scanpipe/tests/data/docker-images.tar.gz-expected-data-2.json b/scanpipe/tests/data/docker-images.tar.gz-expected-data-2.json new file mode 100644 index 0000000000..8cbf180dbc --- /dev/null +++ b/scanpipe/tests/data/docker-images.tar.gz-expected-data-2.json @@ -0,0 +1,394 @@ +[ + { + "sha256": null, + "docker_version": "1.12.6", + "os": "linux", + "os_version": null, + "architecture": "amd64", + "variant": null, + "created": "2017-01-18T09:49:05.022760406Z", + "author": null, + "comment": null, + "labels": { + "com.microscaling.docker.dockerfile": "/Dockerfile", + "com.microscaling.license": "Apache-2.0", + "org.label-schema.build-date": "2017-01-18T09:49:02Z", + "org.label-schema.description": "Our Microscaling Engine provides automation, resilience and efficiency for microservice architectures. Experiment with microscaling at app.microscaling.com.", + "org.label-schema.name": "Microscaling Engine", + "org.label-schema.schema-version": "1.0", + "org.label-schema.url": "https://microscaling.com", + "org.label-schema.vcs-ref": "45b22cb", + "org.label-schema.vcs-url": "https://github.com/microscaling/microscaling.git", + "org.label-schema.vendor": "Microscaling Systems", + "org.label-schema.version": "0.9.1" + }, + "image_format": "docker", + "image_id": "4ac5c955a6dc9dc709fa2ce8dd21c45292afd9e4af38dcd1fb239ed546d19233", + "config_digest": "sha256:4ac5c955a6dc9dc709fa2ce8dd21c45292afd9e4af38dcd1fb239ed546d19233", + "tags": [ + "microscaling/microscaling:latest" + ], + "distro": null, + "layers": [ + { + "extracted_location": "tempdir/7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", + "archive_location": "d388bee71bbf28f77042d89b353bacd14506227a39b6b1098afe457f0a0b608e/layer.tar", + "sha256": "7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "7cbcbac42c44c6c38559e5df3a494f44987333c8023a40fec48df2fce1fc146b", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2016-12-27T18:17:13.762716133Z", + "created_by": "/bin/sh -c #(nop) ADD file:eeed5f514a35d18fcd9cbfe6c40c582211020bffdd53e4799018d33826fe5067 in / ", + "comment": null + }, + { + "extracted_location": "tempdir/d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", + "archive_location": "e439273044a3b34ba206c3598cd007dbba34db5af98d40718d585869af68602e/layer.tar", + "sha256": "d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "d242f1731c55e0f057e183146de867e820dd2ef575125ec36b008340a3acc65e", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-03T13:15:58.410035553Z", + "created_by": "/bin/sh -c apk update && apk upgrade && apk add $BUILD_PACKAGES && rm -rf /var/cache/apk/*", + "comment": null + }, + { + "extracted_location": "tempdir/d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", + "archive_location": "ca473f7411dd74e2d4781fc552d75d3b63c830ff66a253eec6bf5a6261c031b2/layer.tar", + "sha256": "d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "d43ffef6b2712ef8ecdd86866e543b21ef8843742bf7c73a308a973534fa6c3f", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-17T11:17:46.675078318Z", + "created_by": "/bin/sh -c #(nop) COPY multi:0847be51b158e2902e11d406f498a1d8d58ede5fbab01297b446d84b42667433 in / ", + "comment": null + }, + { + "extracted_location": "tempdir/76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", + "archive_location": "325919382e7b94a21ef80452c2fb599f5a03364e8559da0eca1359ad6ac394a0/layer.tar", + "sha256": "76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "76ad2c2330f19d6f16fdf86e7b10cc2c1a8160746ffa1c4e3e46c75661f4bdcd", + "size": 30, + "is_empty_layer": false, + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created": "2017-01-17T11:17:48.829523581Z", + "created_by": "/bin/sh -c chmod +x /microscaling", + "comment": null + } + ], + "history": [ + { + "created": "2016-12-27T18:17:13.762716133Z", + "created_by": "/bin/sh -c #(nop) ADD file:eeed5f514a35d18fcd9cbfe6c40c582211020bffdd53e4799018d33826fe5067 in / " + }, + { + "created": "2017-01-03T13:15:49.512603795Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) MAINTAINER Ross Fairbanks \"ross@microscaling.com\"", + "empty_layer": true + }, + { + "created": "2017-01-03T13:15:49.913789767Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ENV BUILD_PACKAGES=ca-certificates", + "empty_layer": true + }, + { + "created": "2017-01-03T13:15:58.410035553Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c apk update && apk upgrade && apk add $BUILD_PACKAGES && rm -rf /var/cache/apk/*" + }, + { + "created": "2017-01-17T11:17:46.675078318Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) COPY multi:0847be51b158e2902e11d406f498a1d8d58ede5fbab01297b446d84b42667433 in / " + }, + { + "created": "2017-01-17T11:17:48.829523581Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c chmod +x /microscaling" + }, + { + "created": "2017-01-17T11:17:49.322379301Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG BUILD_DATE", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:49.639696203Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VERSION", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:49.991577291Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VCS_URL", + "empty_layer": true + }, + { + "created": "2017-01-17T11:17:50.34992704Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) ARG VCS_REF", + "empty_layer": true + }, + { + "created": "2017-01-18T09:49:04.672769694Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) LABEL org.label-schema.build-date=2017-01-18T09:49:02Z org.label-schema.name=Microscaling Engine org.label-schema.description=Our Microscaling Engine provides automation, resilience and efficiency for microservice architectures. Experiment with microscaling at app.microscaling.com. org.label-schema.url=https://microscaling.com org.label-schema.vcs-url=https://github.com/microscaling/microscaling.git org.label-schema.vcs-ref=45b22cb org.label-schema.vendor=Microscaling Systems org.label-schema.version=0.9.1 org.label-schema.schema-version=1.0 com.microscaling.docker.dockerfile=/Dockerfile com.microscaling.license=Apache-2.0", + "empty_layer": true + }, + { + "created": "2017-01-18T09:49:05.022760406Z", + "author": "Ross Fairbanks \"ross@microscaling.com\"", + "created_by": "/bin/sh -c #(nop) CMD [\"/microscaling\"]", + "empty_layer": true + } + ] + }, + { + "sha256": null, + "docker_version": "19.03.8", + "os": "linux", + "os_version": null, + "architecture": "amd64", + "variant": null, + "created": "2020-10-11T21:21:00.914818853Z", + "author": null, + "comment": null, + "labels": { + "maintainer": "Bjoern Kimminich ", + "org.opencontainers.image.authors": "Bjoern Kimminich ", + "org.opencontainers.image.created": "\u201d2020-10-11T21:19:43Z\u201d", + "org.opencontainers.image.description": "Capture-the-Flag (CTF) environment setup tools for OWASP Juice Shop", + "org.opencontainers.image.documentation": "https://help.owasp-juice.shop/part1/ctf.html", + "org.opencontainers.image.licenses": "MIT", + "org.opencontainers.image.revision": "55df34d", + "org.opencontainers.image.source": "https://github.com/bkimminich/juice-shop-ctf.git", + "org.opencontainers.image.title": "OWASP Juice Shop CTF-Extension", + "org.opencontainers.image.url": "https://owasp-juice.shop", + "org.opencontainers.image.vendor": "Open Web Application Security Project", + "org.opencontainers.image.version": "8.1.2" + }, + "image_format": "docker", + "image_id": "2e92ac90bc1e459ee9d2d72a8bcd3787895d3d7d2d75391e620cea6f20b61315", + "config_digest": "sha256:2e92ac90bc1e459ee9d2d72a8bcd3787895d3d7d2d75391e620cea6f20b61315", + "tags": [ + "bkimminich/juice-shop-ctf:latest" + ], + "distro": null, + "layers": [ + { + "extracted_location": "tempdir/3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "archive_location": "2ba5d0825bcb0d843f975f7ba52db62531dbc73df083fae3d355c1a22d4a4bdc/layer.tar", + "sha256": "3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "3e207b409db364b595ba862cdc12be96dcdad8e36c59a03b7b3b61c946a5741a", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-04-24T01:05:03.608058404Z", + "created_by": "/bin/sh -c #(nop) ADD file:b91adb67b670d3a6ff9463e48b7def903ed516be66fc4282d22c53e41512be49 in / ", + "comment": null + }, + { + "extracted_location": "tempdir/09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", + "archive_location": "c9d345514b7a7f4e4d14748c6a7a6cb408f117fccce87ef80bf72429b5360c44/layer.tar", + "sha256": "09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "09c52b6fbc483eb8e2d244a916da54fb3990cdaa575cab35edfbb27e132929cb", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:14.227103847Z", + "created_by": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps curl && ARCH= && alpineArch=\"$(apk --print-arch)\" && case \"${alpineArch##*-}\" in x86_64) ARCH='x64' CHECKSUM=\"b8b33ec1d2f6463ded7c02bf29ef4141ff4aabb2c0e0ba14cf16ac7c0a6a31a5\" ;; *) ;; esac && if [ -n \"${CHECKSUM}\" ]; then set -eu; curl -fsSLO --compressed \"https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\"; echo \"$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" | sha256sum -c - && tar -xJf \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" -C /usr/local --strip-components=1 --no-same-owner && ln -s /usr/local/bin/node /usr/local/bin/nodejs; else echo \"Building from source\" && apk add --no-cache --virtual .build-deps-full binutils-gold g++ gcc gnupg libgcc linux-headers make python3 && for key in 4ED778F539E3634C779C87C6D7062848A1AB005C 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 1C050899334244A8AF75E53792EF661D867B9DFA 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C DD8F2338BAE7501E3DD5AC78C273792F7D83545D A48C2BEE680E841632CD4E44F07496B3EB3C1762 108F52B48DB57BB0CC439B2997B01419BD92F80A B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps-full && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt; fi && rm -f \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" && apk del .build-deps && node --version && npm --version", + "comment": null + }, + { + "extracted_location": "tempdir/55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", + "archive_location": "cbfc5b91e94f389bd405928c3be3367feea459d5917bde9d8b2792b96ae789ef/layer.tar", + "sha256": "55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "55141db9edb2a13ee593cff8c80e883e672e388c8686fd94a4f2518f21de1d32", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:16.985023204Z", + "created_by": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn && yarn --version", + "comment": null + }, + { + "extracted_location": "tempdir/01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", + "archive_location": "bf1ad815335aca28dc1b64e4a9cad7f0bd9f7e9ee8162dce166150b4f1169b0e/layer.tar", + "sha256": "01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "01f37c950ed43fd0ecc47d0a72949201594f650bd63861cc6e6ac8097ca600bf", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-08T16:23:17.192305843Z", + "created_by": "/bin/sh -c #(nop) COPY file:238737301d47304174e4d24f4def935b29b3069c03c72ae8de97d94624382fce in /usr/local/bin/ ", + "comment": null + }, + { + "extracted_location": "tempdir/08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", + "archive_location": "5b2fc24305d353053e3a6aa7a613e75240e34cfede725f4bd4c466468cd3c48c/layer.tar", + "sha256": "08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "08dc907515cbda226cd872c2c79d087eb226fd27182b6b1315306aade51f963d", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-11T21:20:59.851868447Z", + "created_by": "/bin/sh -c #(nop) COPY dir:5bd634490f1e144602e7b39a1ebcb69af1748a34673299424a7fe6f10334bfa1 in /juice-shop-ctf ", + "comment": null + }, + { + "extracted_location": "tempdir/5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", + "archive_location": "ee512e2eb28e302e80a1e0bfad85ee3b18adc84a80e88ceac4325c13e3e083ec/layer.tar", + "sha256": "5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", + "docker_version": null, + "os": null, + "os_version": null, + "architecture": null, + "variant": null, + "labels": [], + "layer_id": "5b4096031e4780d4c3010335ede79886786ec89d22c2bd85642a30beac682ec9", + "size": 30, + "is_empty_layer": false, + "author": null, + "created": "2020-10-11T21:21:00.668316194Z", + "created_by": "/bin/sh -c #(nop) WORKDIR /data", + "comment": null + } + ], + "history": [ + { + "created": "2020-04-24T01:05:03.608058404Z", + "created_by": "/bin/sh -c #(nop) ADD file:b91adb67b670d3a6ff9463e48b7def903ed516be66fc4282d22c53e41512be49 in / " + }, + { + "created": "2020-04-24T01:05:03.92860976Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\"]", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:07.591276187Z", + "created_by": "/bin/sh -c #(nop) ENV NODE_VERSION=14.13.1", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:14.227103847Z", + "created_by": "/bin/sh -c addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node && apk add --no-cache libstdc++ && apk add --no-cache --virtual .build-deps curl && ARCH= && alpineArch=\"$(apk --print-arch)\" && case \"${alpineArch##*-}\" in x86_64) ARCH='x64' CHECKSUM=\"b8b33ec1d2f6463ded7c02bf29ef4141ff4aabb2c0e0ba14cf16ac7c0a6a31a5\" ;; *) ;; esac && if [ -n \"${CHECKSUM}\" ]; then set -eu; curl -fsSLO --compressed \"https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\"; echo \"$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" | sha256sum -c - && tar -xJf \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" -C /usr/local --strip-components=1 --no-same-owner && ln -s /usr/local/bin/node /usr/local/bin/nodejs; else echo \"Building from source\" && apk add --no-cache --virtual .build-deps-full binutils-gold g++ gcc gnupg libgcc linux-headers make python3 && for key in 4ED778F539E3634C779C87C6D7062848A1AB005C 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 1C050899334244A8AF75E53792EF661D867B9DFA 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C DD8F2338BAE7501E3DD5AC78C273792F7D83545D A48C2BEE680E841632CD4E44F07496B3EB3C1762 108F52B48DB57BB0CC439B2997B01419BD92F80A B9E2F5981AA6E0CD28160D9FF13993A75599653C ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz\" && curl -fsSLO --compressed \"https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc\" && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && grep \" node-v$NODE_VERSION.tar.xz\\$\" SHASUMS256.txt | sha256sum -c - && tar -xf \"node-v$NODE_VERSION.tar.xz\" && cd \"node-v$NODE_VERSION\" && ./configure && make -j$(getconf _NPROCESSORS_ONLN) V= && make install && apk del .build-deps-full && cd .. && rm -Rf \"node-v$NODE_VERSION\" && rm \"node-v$NODE_VERSION.tar.xz\" SHASUMS256.txt.asc SHASUMS256.txt; fi && rm -f \"node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz\" && apk del .build-deps && node --version && npm --version" + }, + { + "created": "2020-10-08T16:23:14.624783745Z", + "created_by": "/bin/sh -c #(nop) ENV YARN_VERSION=1.22.5", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:16.985023204Z", + "created_by": "/bin/sh -c apk add --no-cache --virtual .build-deps-yarn curl gnupg tar && for key in 6A010C5166006599AA17F08146C2130DFD2497F5 ; do gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys \"$key\" || gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys \"$key\" || gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys \"$key\" ; done && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz\" && curl -fsSLO --compressed \"https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc\" && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && mkdir -p /opt && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz && apk del .build-deps-yarn && yarn --version" + }, + { + "created": "2020-10-08T16:23:17.192305843Z", + "created_by": "/bin/sh -c #(nop) COPY file:238737301d47304174e4d24f4def935b29b3069c03c72ae8de97d94624382fce in /usr/local/bin/ " + }, + { + "created": "2020-10-08T16:23:17.37961953Z", + "created_by": "/bin/sh -c #(nop) ENTRYPOINT [\"docker-entrypoint.sh\"]", + "empty_layer": true + }, + { + "created": "2020-10-08T16:23:17.549978283Z", + "created_by": "/bin/sh -c #(nop) CMD [\"node\"]", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:52.959207202Z", + "created_by": "/bin/sh -c #(nop) ARG BUILD_DATE", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:53.20642118Z", + "created_by": "/bin/sh -c #(nop) ARG VCS_REF", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:53.439387368Z", + "created_by": "/bin/sh -c #(nop) LABEL maintainer=Bjoern Kimminich org.opencontainers.image.title=OWASP Juice Shop CTF-Extension org.opencontainers.image.description=Capture-the-Flag (CTF) environment setup tools for OWASP Juice Shop org.opencontainers.image.authors=Bjoern Kimminich org.opencontainers.image.vendor=Open Web Application Security Project org.opencontainers.image.documentation=https://help.owasp-juice.shop/part1/ctf.html org.opencontainers.image.licenses=MIT org.opencontainers.image.version=8.1.2 org.opencontainers.image.url=https://owasp-juice.shop org.opencontainers.image.source=https://github.com/bkimminich/juice-shop-ctf.git org.opencontainers.image.revision=55df34d org.opencontainers.image.created=\u201d2020-10-11T21:19:43Z\u201d", + "empty_layer": true + }, + { + "created": "2020-10-11T21:20:59.851868447Z", + "created_by": "/bin/sh -c #(nop) COPY dir:5bd634490f1e144602e7b39a1ebcb69af1748a34673299424a7fe6f10334bfa1 in /juice-shop-ctf " + }, + { + "created": "2020-10-11T21:21:00.397239764Z", + "created_by": "/bin/sh -c #(nop) VOLUME [/data]", + "empty_layer": true + }, + { + "created": "2020-10-11T21:21:00.668316194Z", + "created_by": "/bin/sh -c #(nop) WORKDIR /data" + }, + { + "created": "2020-10-11T21:21:00.914818853Z", + "created_by": "/bin/sh -c #(nop) ENTRYPOINT [\"npx\" \"/juice-shop-ctf/bin/juice-shop-ctf.js\"]", + "empty_layer": true + } + ] + } +] \ No newline at end of file diff --git a/scanpipe/tests/test_pipes_docker.py b/scanpipe/tests/test_pipes_docker.py new file mode 100644 index 0000000000..63171c9b8b --- /dev/null +++ b/scanpipe/tests/test_pipes_docker.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import json +import tempfile +from pathlib import Path + +from django.apps import apps +from django.test import TestCase + +from scanpipe.pipes import docker + +scanpipe_app = apps.get_app_config("scanpipe") + + +class ScanPipeDcokerPipesTest(TestCase): + data_location = Path(__file__).parent / "data" + + def test_docker__get_image_data_contains_layers__with_relative_paths( + self, regen=False + ): + extract_target = str(Path(tempfile.mkdtemp()) / "tempdir") + + input_tarball = str(self.data_location / "docker-images.tar.gz") + + # extract the image first + images, errors = docker.extract_image_from_tarball( + input_tarball, + extract_target, + verify=False, + ) + assert not errors + + images_data = [docker.get_image_data(i) for i in images] + results = json.dumps(images_data, indent=2) + + expected_location = str( + self.data_location / "docker-images.tar.gz-expected-data-1.json" + ) + + if regen: + expected = results + + with open(expected_location, "w") as out: + out.write(expected) + else: + with open(expected_location) as inp: + expected = inp.read() + + self.assertEqual(results, expected) + + # extract the layers second + errors = docker.extract_layers_from_images_to_base_path( + base_path=extract_target, + images=images, + ) + assert not errors + + images_data = [docker.get_image_data(i) for i in images] + results = json.dumps(images_data, indent=2) + + expected_location = str( + self.data_location / "docker-images.tar.gz-expected-data-2.json" + ) + if regen: + expected = results + + with open(expected_location, "w") as out: + out.write(expected) + else: + with open(expected_location) as inp: + expected = inp.read() + + self.assertEqual(results, expected) From 6e2fb32888322e875289fbcfe8bc1c8cba2836fd Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 26 Oct 2021 18:29:18 +0200 Subject: [PATCH 3/5] Report relative path for layers Use latest container-inspector Reference: https://github.com/nexB/scancode.io/issues/175 Reported-by: Sameer1046 @Sameer1046 Signed-off-by: Philippe Ombredanne --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3af58e6eec..93d7c84188 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,7 +70,7 @@ install_requires = # WSGI server gunicorn==20.1.0 # Docker - container_inspector==21.6.10 + container_inspector==30.0.0 # ScanCode-toolkit scancode-toolkit[packages]==30.1.0 extractcode[full]==21.7.23 From baef98b62d44ea75d90b85ad98c2a6597baca726 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Mon, 15 Nov 2021 14:24:24 +0400 Subject: [PATCH 4/5] Fix typos, code consistency, formatting, and unit test #175 Signed-off-by: Thomas Druez --- scanpipe/pipes/docker.py | 9 ++-- scanpipe/tests/test_pipes.py | 12 ----- scanpipe/tests/test_pipes_docker.py | 75 ++++++++++++++--------------- 3 files changed, 39 insertions(+), 57 deletions(-) diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index 0d4700b0de..3cfadccfd5 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -79,17 +79,13 @@ def extract_layers_from_images(project, images): Returns an `errors` list of error messages that may occur during the extraction. """ - return extract_layers_from_images_to_base_path( base_path=project.codebase_path, images=images, ) -def extract_layers_from_images_to_base_path( - base_path, - images, -): +def extract_layers_from_images_to_base_path(base_path, images): """ Extracts all layers from the provided `images` into the `base_path` work directory. @@ -99,6 +95,7 @@ def extract_layers_from_images_to_base_path( """ errors = [] base_path = Path(base_path) + for image in images: image_dirname = Path(image.extracted_location).name target_path = base_path / image_dirname @@ -113,6 +110,8 @@ def extract_layers_from_images_to_base_path( errors.extend(extract_errors) layer.extracted_location = str(extract_target) + return errors + def get_image_data(image, layer_path_segments=2): """ diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index 2ca184000f..484bc50a05 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -40,7 +40,6 @@ from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.pipes import codebase -from scanpipe.pipes import docker from scanpipe.pipes import fetch from scanpipe.pipes import filename_now from scanpipe.pipes import make_codebase_resource @@ -744,17 +743,6 @@ def test_scanpipe_pipes_fetch_fetch_urls(self, mock_get): self.assertEqual(2, len(errors)) self.assertEqual(urls, errors) - def test_scanpipe_pipes_docker_tag_whiteout_codebase_resources(self): - p1 = Project.objects.create(name="Analysis") - resource1 = CodebaseResource.objects.create(project=p1, path="filename.ext") - resource2 = CodebaseResource.objects.create(project=p1, name=".wh.filename2") - - docker.tag_whiteout_codebase_resources(p1) - resource1.refresh_from_db() - resource2.refresh_from_db() - self.assertEqual("", resource1.status) - self.assertEqual("ignored-whiteout", resource2.status) - def test_scanpipe_pipes_rootfs_from_project_codebase_class_method(self): p1 = Project.objects.create(name="Analysis") root_filesystems = list(rootfs.RootFs.from_project_codebase(p1)) diff --git a/scanpipe/tests/test_pipes_docker.py b/scanpipe/tests/test_pipes_docker.py index 63171c9b8b..1769df4bc4 100644 --- a/scanpipe/tests/test_pipes_docker.py +++ b/scanpipe/tests/test_pipes_docker.py @@ -27,67 +27,62 @@ from django.apps import apps from django.test import TestCase +from scanpipe.models import CodebaseResource +from scanpipe.models import Project from scanpipe.pipes import docker scanpipe_app = apps.get_app_config("scanpipe") -class ScanPipeDcokerPipesTest(TestCase): - data_location = Path(__file__).parent / "data" +class ScanPipeDockerPipesTest(TestCase): + data_path = Path(__file__).parent / "data" - def test_docker__get_image_data_contains_layers__with_relative_paths( - self, regen=False - ): - extract_target = str(Path(tempfile.mkdtemp()) / "tempdir") + def assertResultsEqual(self, expected_file, results, regen=False): + """ + Set `regen` to True to regenerate the expected results. + """ + if regen: + expected_file.write_text(results) - input_tarball = str(self.data_location / "docker-images.tar.gz") + expected_data = expected_file.read_text() + self.assertEqual(expected_data, results) + + def test_pipes_docker_get_image_data_contains_layers_with_relative_paths(self): + extract_target = str(Path(tempfile.mkdtemp()) / "tempdir") + input_tarball = str(self.data_path / "docker-images.tar.gz") - # extract the image first + # Extract the image first images, errors = docker.extract_image_from_tarball( input_tarball, extract_target, verify=False, ) - assert not errors + self.assertEqual([], errors) images_data = [docker.get_image_data(i) for i in images] results = json.dumps(images_data, indent=2) + expected_location = self.data_path / "docker-images.tar.gz-expected-data-1.json" + self.assertResultsEqual(expected_location, results, regen=False) - expected_location = str( - self.data_location / "docker-images.tar.gz-expected-data-1.json" - ) - - if regen: - expected = results - - with open(expected_location, "w") as out: - out.write(expected) - else: - with open(expected_location) as inp: - expected = inp.read() - - self.assertEqual(results, expected) - - # extract the layers second + # Extract the layers second errors = docker.extract_layers_from_images_to_base_path( base_path=extract_target, images=images, ) - assert not errors + self.assertEqual([], errors) images_data = [docker.get_image_data(i) for i in images] results = json.dumps(images_data, indent=2) - - expected_location = str( - self.data_location / "docker-images.tar.gz-expected-data-2.json" - ) - if regen: - expected = results - - with open(expected_location, "w") as out: - out.write(expected) - else: - with open(expected_location) as inp: - expected = inp.read() - - self.assertEqual(results, expected) + expected_location = self.data_path / "docker-images.tar.gz-expected-data-2.json" + self.assertResultsEqual(expected_location, results, regen=False) + + def test_pipes_docker_tag_whiteout_codebase_resources(self): + p1 = Project.objects.create(name="Analysis") + resource1 = CodebaseResource.objects.create(project=p1, path="filename.ext") + resource2 = CodebaseResource.objects.create(project=p1, name=".wh.filename2") + + docker.tag_whiteout_codebase_resources(p1) + resource1.refresh_from_db() + resource2.refresh_from_db() + self.assertEqual("", resource1.status) + self.assertEqual("ignored-whiteout", resource2.status) From 0585a65047a48ac1b15d7d0a91b130b5f2d86197 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Mon, 15 Nov 2021 15:50:51 +0400 Subject: [PATCH 5/5] Add CHANGELOG entry #175 Signed-off-by: Thomas Druez --- CHANGELOG.rst | 3 +++ scanpipe/pipes/docker.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 75fd6bbd67..6d850893cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog Unreleased ---------- +- Include layers in docker image data. + https://github.com/nexB/scancode.io/issues/175 + - Fix a server error on resource details view when the compliance alert is "missing". https://github.com/nexB/scancode.io/issues/344 diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index 3cfadccfd5..fa7a999564 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -101,7 +101,6 @@ def extract_layers_from_images_to_base_path(base_path, images): target_path = base_path / image_dirname for layer in image.layers: - # TODO: using the layer may be less useful than using the layer sha256 extract_target = target_path / layer.layer_id extract_errors = extract_archive( location=layer.archive_location,