From 8180ef4388f2249f4ab7920d8e6ec9f8b6afd508 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 1 Dec 2023 12:00:06 +1100 Subject: [PATCH 1/4] Fix vendored deps for windows --- pkg/osvscanner/osvscanner.go | 111 +-------------------------- pkg/osvscanner/vendored_libs.go | 128 ++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 107 deletions(-) create mode 100644 pkg/osvscanner/vendored_libs.go diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 03a7aeddb9..90faa85206 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -1,11 +1,9 @@ package osvscanner import ( - "bufio" - "crypto/md5" //nolint:gosec + "bufio" //nolint:gosec "errors" "fmt" - "io/fs" "os" "os/exec" "path" @@ -70,22 +68,6 @@ var VulnerabilitiesFoundErr = errors.New("vulnerabilities found") //nolint:errname,stylecheck // Would require version bump to change var OnlyUncalledVulnerabilitiesFoundErr = errors.New("only uncalled vulnerabilities found") -var ( - vendoredLibNames = map[string]struct{}{ - "3rdparty": {}, - "dep": {}, - "deps": {}, - "thirdparty": {}, - "third-party": {}, - "third_party": {}, - "libs": {}, - "external": {}, - "externals": {}, - "vendor": {}, - "vendored": {}, - } -) - const ( // This value may need to be tweaked, or be provided as a configurable flag. determineVersionThreshold = 0.15 @@ -141,6 +123,7 @@ func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useG } } + // Scan git commit hash if !skipGit && info.IsDir() && info.Name() == ".git" { pkgs, err := scanGit(r, filepath.Dir(path)+"/") if err != nil { @@ -152,6 +135,7 @@ func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useG return filepath.SkipDir } + // Scan lockfiles if !info.IsDir() { if extractor, _ := lockfile.FindExtractor(path, ""); extractor != nil { pkgs, err := scanLockfile(r, path, "") @@ -167,6 +151,7 @@ func scanDir(r reporter.Reporter, dir string, skipGit bool, recursive bool, useG scannedPackages = append(scannedPackages, pkgs...) } + // Scan vendored directories if info.IsDir() && !compareOffline { if _, ok := vendoredLibNames[strings.ToLower(filepath.Base(path))]; ok { pkgs, err := scanDirWithVendoredLibs(r, path) @@ -225,94 +210,6 @@ func parseGitIgnores(path string) (*gitIgnoreMatcher, error) { return &gitIgnoreMatcher{matcher: matcher, repoPath: repopath}, nil } -func queryDetermineVersions(repoDir string) (*osv.DetermineVersionResponse, error) { - fileExts := []string{ - ".hpp", - ".h", - ".hh", - ".cc", - ".c", - ".cpp", - } - - var hashes []osv.DetermineVersionHash - if err := filepath.Walk(repoDir, func(p string, info fs.FileInfo, err error) error { - if info.IsDir() { - if _, err := os.Stat(filepath.Join(p, ".git")); err == nil { - // Found a git repo, stop here as otherwise we may get duplicated - // results with our regular git commit scanning. - return filepath.SkipDir - } - if _, ok := vendoredLibNames[strings.ToLower(info.Name())]; ok { - // Ignore nested vendored libraries, as they can cause bad matches. - return filepath.SkipDir - } - - return nil - } - for _, ext := range fileExts { - if filepath.Ext(p) == ext { - buf, err := os.ReadFile(p) - if err != nil { - return err - } - hash := md5.Sum(buf) //nolint:gosec - hashes = append(hashes, osv.DetermineVersionHash{ - Path: strings.ReplaceAll(p, repoDir, ""), - Hash: hash[:], - }) - if len(hashes) > maxDetermineVersionFiles { - return errors.New("too many files to hash") - } - } - } - - return nil - }); err != nil { - return nil, fmt.Errorf("failed during hashing: %w", err) - } - - result, err := osv.MakeDetermineVersionRequest(filepath.Base(repoDir), hashes) - if err != nil { - return nil, fmt.Errorf("failed to determine versions: %w", err) - } - - return result, nil -} - -func scanDirWithVendoredLibs(r reporter.Reporter, path string) ([]scannedPackage, error) { - r.PrintText(fmt.Sprintf("Scanning directory for vendored libs: %s\n", path)) - entries, err := os.ReadDir(path) - if err != nil { - return nil, err - } - - var packages []scannedPackage - for _, entry := range entries { - if !entry.IsDir() { - continue - } - - libPath := filepath.Join(path, entry.Name()) - - r.PrintText(fmt.Sprintf("Scanning potential vendored dir: %s\n", libPath)) - // TODO: make this a goroutine to parallelise this operation - results, err := queryDetermineVersions(libPath) - if err != nil { - r.PrintText(fmt.Sprintf("Error scanning sub-directory '%s' with error: %v", libPath, err)) - continue - } - - if len(results.Matches) > 0 && results.Matches[0].Score > determineVersionThreshold { - match := results.Matches[0] - r.PrintText(fmt.Sprintf("Identified %s as %s at %s.\n", libPath, match.RepoInfo.Address, match.RepoInfo.Commit)) - packages = append(packages, createCommitQueryPackage(match.RepoInfo.Commit, libPath)) - } - } - - return packages, nil -} - // gitIgnoreMatcher.match will return true if the file/directory matches a gitignore entry // i.e. true if it should be ignored func (m *gitIgnoreMatcher) match(absPath string, isDir bool) (bool, error) { diff --git a/pkg/osvscanner/vendored_libs.go b/pkg/osvscanner/vendored_libs.go new file mode 100644 index 0000000000..51f67fcac6 --- /dev/null +++ b/pkg/osvscanner/vendored_libs.go @@ -0,0 +1,128 @@ +package osvscanner + +import ( + "bytes" + "crypto/md5" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/google/osv-scanner/pkg/osv" + "github.com/google/osv-scanner/pkg/reporter" +) + +var ( + vendoredLibNames = map[string]struct{}{ + "3rdparty": {}, + "dep": {}, + "deps": {}, + "thirdparty": {}, + "third-party": {}, + "third_party": {}, + "libs": {}, + "external": {}, + "externals": {}, + "vendor": {}, + "vendored": {}, + } +) + +func scanDirWithVendoredLibs(r reporter.Reporter, path string) ([]scannedPackage, error) { + r.PrintText(fmt.Sprintf("Scanning directory for vendored libs: %s\n", path)) + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + var packages []scannedPackage + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + libPath := filepath.Join(path, entry.Name()) + + r.PrintText(fmt.Sprintf("Scanning potential vendored dir: %s\n", libPath)) + // TODO: make this a goroutine to parallelize this operation + results, err := queryDetermineVersions(libPath) + if err != nil { + r.PrintText(fmt.Sprintf("Error scanning sub-directory '%s' with error: %v", libPath, err)) + continue + } + + if len(results.Matches) > 0 && results.Matches[0].Score > determineVersionThreshold { + match := results.Matches[0] + r.PrintText(fmt.Sprintf("Identified %s as %s at %s.\n", libPath, match.RepoInfo.Address, match.RepoInfo.Commit)) + packages = append(packages, createCommitQueryPackage(match.RepoInfo.Commit, libPath)) + } + } + + return packages, nil +} + +func queryDetermineVersions(repoDir string) (*osv.DetermineVersionResponse, error) { + fileExts := []string{ + ".hpp", + ".h", + ".hh", + ".cc", + ".c", + ".cpp", + } + + var hashes []osv.DetermineVersionHash + if err := filepath.Walk(repoDir, func(p string, info fs.FileInfo, err error) error { + if info.IsDir() { + if _, err := os.Stat(filepath.Join(p, ".git")); err == nil { + // Found a git repo, stop here as otherwise we may get duplicated + // results with our regular git commit scanning. + return filepath.SkipDir + } + if _, ok := vendoredLibNames[strings.ToLower(info.Name())]; ok { + // Ignore nested vendored libraries, as they can cause bad matches. + return filepath.SkipDir + } + + return nil + } + + windowsEnding := []byte("\r\n") + unixEnding := []byte("\n") + + for _, ext := range fileExts { + if filepath.Ext(p) != ext { + continue + } + + buf, err := os.ReadFile(p) + if err != nil { + return err + } + + buf = bytes.ReplaceAll(buf, windowsEnding, unixEnding) + + hash := md5.Sum(buf) //nolint:gosec + hashes = append(hashes, osv.DetermineVersionHash{ + Path: strings.ReplaceAll(p, repoDir, ""), + Hash: hash[:], + }) + if len(hashes) > maxDetermineVersionFiles { + return errors.New("too many files to hash") + } + } + + return nil + }); err != nil { + return nil, fmt.Errorf("failed during hashing: %w", err) + } + + result, err := osv.MakeDetermineVersionRequest(filepath.Base(repoDir), hashes) + if err != nil { + return nil, fmt.Errorf("failed to determine versions: %w", err) + } + + return result, nil +} From 86c959deb31b896febaa01c147d1fd1bdd3d1a00 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 1 Dec 2023 14:23:34 +1100 Subject: [PATCH 2/4] Add test for scanDirVendoredLib --- .../fixtures/example-vendor/bsdiff/LICENCE | 24 + .../fixtures/example-vendor/bsdiff/README.md | 135 +++++ .../fixtures/example-vendor/bsdiff/bsdiff.c | 461 ++++++++++++++++++ .../fixtures/example-vendor/bsdiff/bspatch.c | 198 ++++++++ pkg/osvscanner/osvscanner_internal_test.go | 4 +- pkg/osvscanner/vendored_libs_test.go | 52 ++ 6 files changed, 872 insertions(+), 2 deletions(-) create mode 100644 pkg/osvscanner/fixtures/example-vendor/bsdiff/LICENCE create mode 100644 pkg/osvscanner/fixtures/example-vendor/bsdiff/README.md create mode 100644 pkg/osvscanner/fixtures/example-vendor/bsdiff/bsdiff.c create mode 100644 pkg/osvscanner/fixtures/example-vendor/bsdiff/bspatch.c create mode 100644 pkg/osvscanner/vendored_libs_test.go diff --git a/pkg/osvscanner/fixtures/example-vendor/bsdiff/LICENCE b/pkg/osvscanner/fixtures/example-vendor/bsdiff/LICENCE new file mode 100644 index 0000000000..5d40665778 --- /dev/null +++ b/pkg/osvscanner/fixtures/example-vendor/bsdiff/LICENCE @@ -0,0 +1,24 @@ + Copyright 2003-2005 Colin Percival + Copyright 2012 Matthew Endsley + All rights reserved + + Redistribution and use in source and binary forms, with or without + modification, are permitted providing that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/osvscanner/fixtures/example-vendor/bsdiff/README.md b/pkg/osvscanner/fixtures/example-vendor/bsdiff/README.md new file mode 100644 index 0000000000..799716222d --- /dev/null +++ b/pkg/osvscanner/fixtures/example-vendor/bsdiff/README.md @@ -0,0 +1,135 @@ +bsdiff/bspatch +============== +bsdiff and bspatch are libraries for building and applying patches to binary +files. + +The original algorithm and implementation was developed by Colin Percival. The +algorithm is detailed in his doctoral thesis: +. For more information visit his +website at . + +I maintain this project seperately from Colin's work, with the goal of making +the core functionality easily embedable in existing projects. + +Contact +------- +[@MatthewEndsley](https://twitter.com/#!/MatthewEndsley) + + +License +------- +Copyright 2003-2005 Colin Percival +Copyright 2012 Matthew Endsley + +This project is governed by the BSD 2-clause license. For details see the file +titled LICENSE in the project root folder. + +Overview +-------- +There are two separate libraries in the project, bsdiff and bspatch. Each are +self contained in bsdiff.c and bspatch.c The easiest way to integrate is to +simply copy the c file to your source folder and build it. + +The overarching goal was to modify the original bsdiff/bspatch code from Colin +and eliminate external dependencies and provide a simple interface to the core +functionality. + +You can define `BSDIFF_HEADER_ONLY` or `BSPATCH_HEADER_ONLY` to only include +the header parts of the file. If including a `.c` file makes you feel really +dirty you can copy paste the header portion at the top of the file into your own +`.h` file. + +I've exposed relevant functions via the `_stream` classes. The only external +dependency not exposed is `memcmp` in `bsdiff`. + +This library generates patches that are not compatible with the original bsdiff +tool. The impompatibilities were motivated by the patching needs for the game +AirMech and the following requirements: + +* Eliminate/minimize any seek operations when applying patches +* Eliminate any required disk I/O and support embedded streams +* Ability to easily embed the routines as a library instead of an external binary +* Compile+run on all platforms we use to build the game (Windows, Linux, NaCl, OSX) + +Compiling +--------- +The libraries should compile warning free in any moderately recent version of +gcc. The project uses `` which is technically a C99 file and not +available in Microsoft Visual Studio. The easiest solution here is to use the +msinttypes version of stdint.h from . +The direct link for the lazy people is: +. + +If your compiler does not provide an implementation of `` you can +remove the header from the bsdiff/bspatch files and provide your own typedefs +for the following symbols: `uint8_t`, `uint64_t` and `int64_t`. + +Examples +-------- +Each project has an optional main function that serves as an example for using +the library. Simply defined `BSDIFF_EXECUTABLE` or `BSPATCH_EXECUTABLE` to +enable building the standalone tools. + +Reference +--------- +### bsdiff + + struct bsdiff_stream + { + void* opaque; + void* (*malloc)(size_t size); + void (*free)(void* ptr); + int (*write)(struct bsdiff_stream* stream, + const void* buffer, int size); + }; + + int bsdiff(const uint8_t* old, int64_t oldsize, const uint8_t* new, + int64_t newsize, struct bsdiff_stream* stream); + + +In order to use `bsdiff`, you need to define functions for allocating memory and +writing binary data. This behavior is controlled by the `stream` parameted +passed to to `bsdiff(...)`. + +The `opaque` field is never read or modified from within the `bsdiff` function. +The caller can use this field to store custom state data needed for the callback +functions. + +The `malloc` and `free` members should point to functions that behave like the +standard `malloc` and `free` C functions. + +The `write` function is called by bsdiff to write a block of binary data to the +stream. The return value for `write` should be `0` on success and non-zero if +the callback failed to write all data. In the default example, bzip2 is used to +compress output data. + +`bsdiff` returns `0` on success and `-1` on failure. + +### bspatch + + struct bspatch_stream + { + void* opaque; + int (*read)(const struct bspatch_stream* stream, + void* buffer, int length); + }; + + int bspatch(const uint8_t* old, int64_t oldsize, uint8_t* new, + int64_t newsize, struct bspatch_stream* stream); + +The `bspatch` function transforms the data for a file using data generated from +`bsdiff`. The caller takes care of loading the old file and allocating space for +new file data. The `stream` parameter controls the process for reading binary +patch data. + +The `opaque` field is never read or modified from within the bspatch function. +The caller can use this field to store custom state data needed for the read +function. + +The `read` function is called by `bspatch` to read a block of binary data from +the stream. The return value for `read` should be `0` on success and non-zero +if the callback failed to read the requested amount of data. In the default +example, bzip2 is used to decompress input data. + +`bspatch` returns `0` on success and `-1` on failure. On success, `new` contains +the data for the patched file. diff --git a/pkg/osvscanner/fixtures/example-vendor/bsdiff/bsdiff.c b/pkg/osvscanner/fixtures/example-vendor/bsdiff/bsdiff.c new file mode 100644 index 0000000000..3857c017e4 --- /dev/null +++ b/pkg/osvscanner/fixtures/example-vendor/bsdiff/bsdiff.c @@ -0,0 +1,461 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +struct bsdiff_stream +{ + void* opaque; + + void* (*malloc)(size_t size); + void (*free)(void* ptr); + int (*write)(struct bsdiff_stream* stream, const void* buffer, int size); +}; + +int bsdiff(const uint8_t* old, int64_t oldsize, const uint8_t* new, int64_t newsize, struct bsdiff_stream* stream); + +#if !defined(BSDIFF_HEADER_ONLY) + +#include +#include + +#define MIN(x,y) (((x)<(y)) ? (x) : (y)) + +static void split(int64_t *I,int64_t *V,int64_t start,int64_t len,int64_t h) +{ + int64_t i,j,k,x,tmp,jj,kk; + + if(len<16) { + for(k=start;kstart) split(I,V,start,jj-start,h); + + for(i=0;ikk) split(I,V,kk,start+len-kk,h); +} + +static void qsufsort(int64_t *I,int64_t *V,const uint8_t *old,int64_t oldsize) +{ + int64_t buckets[256]; + int64_t i,h,len; + + for(i=0;i<256;i++) buckets[i]=0; + for(i=0;i0;i--) buckets[i]=buckets[i-1]; + buckets[0]=0; + + for(i=0;iy) { + *pos=I[st]; + return x; + } else { + *pos=I[en]; + return y; + } + }; + + x=st+(en-st)/2; + if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { + return search(I,old,oldsize,new,newsize,x,en,pos); + } else { + return search(I,old,oldsize,new,newsize,st,x,pos); + }; +} + +static void offtout(int64_t x,uint8_t *buf) +{ + int64_t y; + + if(x<0) y=-x; else y=x; + + buf[0]=y%256;y-=buf[0]; + y=y/256;buf[1]=y%256;y-=buf[1]; + y=y/256;buf[2]=y%256;y-=buf[2]; + y=y/256;buf[3]=y%256;y-=buf[3]; + y=y/256;buf[4]=y%256;y-=buf[4]; + y=y/256;buf[5]=y%256;y-=buf[5]; + y=y/256;buf[6]=y%256;y-=buf[6]; + y=y/256;buf[7]=y%256; + + if(x<0) buf[7]|=0x80; +} + +static int64_t writedata(struct bsdiff_stream* stream, const void* buffer, int64_t length) +{ + int64_t result = 0; + + while (length > 0) + { + const int smallsize = (int)MIN(length, INT_MAX); + const int writeresult = stream->write(stream, buffer, smallsize); + if (writeresult == -1) + { + return -1; + } + + result += writeresult; + length -= smallsize; + buffer = (uint8_t*)buffer + smallsize; + } + + return result; +} + +struct bsdiff_request +{ + const uint8_t* old; + int64_t oldsize; + const uint8_t* new; + int64_t newsize; + struct bsdiff_stream* stream; + int64_t *I; + uint8_t *buffer; +}; + +static int bsdiff_internal(const struct bsdiff_request req) +{ + int64_t *I,*V; + int64_t scan,pos,len; + int64_t lastscan,lastpos,lastoffset; + int64_t oldscore,scsc; + int64_t s,Sf,lenf,Sb,lenb; + int64_t overlap,Ss,lens; + int64_t i; + uint8_t *buffer; + uint8_t buf[8 * 3]; + + if((V=req.stream->malloc((req.oldsize+1)*sizeof(int64_t)))==NULL) return -1; + I = req.I; + + qsufsort(I,V,req.old,req.oldsize); + req.stream->free(V); + + buffer = req.buffer; + + /* Compute the differences, writing ctrl as we go */ + scan=0;len=0;pos=0; + lastscan=0;lastpos=0;lastoffset=0; + while(scanoldscore+8)) break; + + if((scan+lastoffsetSf*2-lenf) { Sf=s; lenf=i; }; + }; + + lenb=0; + if(scan=lastscan+i)&&(pos>=i);i++) { + if(req.old[pos-i]==req.new[scan-i]) s++; + if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; + }; + }; + + if(lastscan+lenf>scan-lenb) { + overlap=(lastscan+lenf)-(scan-lenb); + s=0;Ss=0;lens=0; + for(i=0;iSs) { Ss=s; lens=i+1; }; + }; + + lenf+=lens-overlap; + lenb-=lens; + }; + + offtout(lenf,buf); + offtout((scan-lenb)-(lastscan+lenf),buf+8); + offtout((pos-lenb)-(lastpos+lenf),buf+16); + + /* Write control data */ + if (writedata(req.stream, buf, sizeof(buf))) + return -1; + + /* Write diff data */ + for(i=0;imalloc((oldsize+1)*sizeof(int64_t)))==NULL) + return -1; + + if((req.buffer=stream->malloc(newsize+1))==NULL) + { + stream->free(req.I); + return -1; + } + + req.old = old; + req.oldsize = oldsize; + req.new = new; + req.newsize = newsize; + req.stream = stream; + + result = bsdiff_internal(req); + + stream->free(req.buffer); + stream->free(req.I); + + return result; +} + +#if defined(BSDIFF_EXECUTABLE) + +#include + +#include +#include +#include +#include +#include +#include + +static int bz2_write(struct bsdiff_stream* stream, const void* buffer, int size) +{ + int bz2err; + BZFILE* bz2; + + bz2 = (BZFILE*)stream->opaque; + BZ2_bzWrite(&bz2err, bz2, (void*)buffer, size); + if (bz2err != BZ_STREAM_END && bz2err != BZ_OK) + return -1; + + return 0; +} + +int main(int argc,char *argv[]) +{ + int fd; + int bz2err; + uint8_t *old,*new; + off_t oldsize,newsize; + uint8_t buf[8]; + FILE * pf; + struct bsdiff_stream stream; + BZFILE* bz2; + + memset(&bz2, 0, sizeof(bz2)); + stream.malloc = malloc; + stream.free = free; + stream.write = bz2_write; + + if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); + + /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if(((fd=open(argv[1],O_RDONLY,0))<0) || + ((oldsize=lseek(fd,0,SEEK_END))==-1) || + ((old=malloc(oldsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,old,oldsize)!=oldsize) || + (close(fd)==-1)) err(1,"%s",argv[1]); + + + /* Allocate newsize+1 bytes instead of newsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if(((fd=open(argv[2],O_RDONLY,0))<0) || + ((newsize=lseek(fd,0,SEEK_END))==-1) || + ((new=malloc(newsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,new,newsize)!=newsize) || + (close(fd)==-1)) err(1,"%s",argv[2]); + + /* Create the patch file */ + if ((pf = fopen(argv[3], "w")) == NULL) + err(1, "%s", argv[3]); + + /* Write header (signature+newsize)*/ + offtout(newsize, buf); + if (fwrite("ENDSLEY/BSDIFF43", 16, 1, pf) != 1 || + fwrite(buf, sizeof(buf), 1, pf) != 1) + err(1, "Failed to write header"); + + + if (NULL == (bz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0))) + errx(1, "BZ2_bzWriteOpen, bz2err=%d", bz2err); + + stream.opaque = bz2; + if (bsdiff(old, oldsize, new, newsize, &stream)) + err(1, "bsdiff"); + + BZ2_bzWriteClose(&bz2err, bz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + err(1, "BZ2_bzWriteClose, bz2err=%d", bz2err); + + if (fclose(pf)) + err(1, "fclose"); + + /* Free the memory we used */ + free(old); + free(new); + + return 0; +} + +#endif + +#endif diff --git a/pkg/osvscanner/fixtures/example-vendor/bsdiff/bspatch.c b/pkg/osvscanner/fixtures/example-vendor/bsdiff/bspatch.c new file mode 100644 index 0000000000..7de0a08a48 --- /dev/null +++ b/pkg/osvscanner/fixtures/example-vendor/bsdiff/bspatch.c @@ -0,0 +1,198 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +struct bspatch_stream +{ + void* opaque; + int (*read)(const struct bspatch_stream* stream, void* buffer, int length); +}; + +int bspatch(const uint8_t* old, int64_t oldsize, uint8_t* new, int64_t newsize, struct bspatch_stream* stream); + +#if !defined(BSPATCH_HEADER_ONLY) + +static int64_t offtin(uint8_t *buf) +{ + int64_t y; + + y=buf[7]&0x7F; + y=y*256;y+=buf[6]; + y=y*256;y+=buf[5]; + y=y*256;y+=buf[4]; + y=y*256;y+=buf[3]; + y=y*256;y+=buf[2]; + y=y*256;y+=buf[1]; + y=y*256;y+=buf[0]; + + if(buf[7]&0x80) y=-y; + + return y; +} + +int bspatch(const uint8_t* old, int64_t oldsize, uint8_t* new, int64_t newsize, struct bspatch_stream* stream) +{ + uint8_t buf[8]; + int64_t oldpos,newpos; + int64_t ctrl[3]; + int64_t i; + + oldpos=0;newpos=0; + while(newposread(stream, buf, 8)) + return -1; + ctrl[i]=offtin(buf); + }; + + /* Sanity-check */ + if(newpos+ctrl[0]>newsize) + return -1; + + /* Read diff string */ + if (stream->read(stream, new + newpos, ctrl[0])) + return -1; + + /* Add old data to diff string */ + for(i=0;i=0) && (oldpos+inewsize) + return -1; + + /* Read extra string */ + if (stream->read(stream, new + newpos, ctrl[1])) + return -1; + + /* Adjust pointers */ + newpos+=ctrl[1]; + oldpos+=ctrl[2]; + }; + + return 0; +} + +#if defined(BSPATCH_EXECUTABLE) + +#include +#include +#include +#include +#include +#include +#include +#include + +static int bz2_read(const struct bspatch_stream* stream, void* buffer, int length) +{ + int n; + int bz2err; + BZFILE* bz2; + + bz2 = (BZFILE*)stream->opaque; + n = BZ2_bzRead(&bz2err, bz2, buffer, length); + if (n != length) + return -1; + + return 0; +} + +int main(int argc,char * argv[]) +{ + FILE * f; + int fd; + int bz2err; + uint8_t header[24]; + uint8_t *old, *new; + int64_t oldsize, newsize; + BZFILE* bz2; + struct bspatch_stream stream; + + if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); + + /* Open patch file */ + if ((f = fopen(argv[3], "r")) == NULL) + err(1, "fopen(%s)", argv[3]); + + /* Read header */ + if (fread(header, 1, 16, f) != 16) { + if (feof(f)) + errx(1, "Corrupt patch\n"); + err(1, "fread(%s)", argv[3]); + } + + /* Check for appropriate magic */ + if (memcmp(header, "ENDSLEY/BSDIFF43", 16) != 0) + errx(1, "Corrupt patch\n"); + + /* Read lengths from header */ + newsize=offtin(header+16) + if(newsize<0) + errx(1,"Corrupt patch\n"); + + /* Close patch file and re-open it via libbzip2 at the right places */ + if(((fd=open(argv[1],O_RDONLY,0))<0) || + ((oldsize=lseek(fd,0,SEEK_END))==-1) || + ((old=malloc(oldsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,old,oldsize)!=oldsize) || + (close(fd)==-1)) err(1,"%s",argv[1]); + if((new=malloc(newsize+1))==NULL) err(1,NULL); + + if (NULL == (bz2 = BZ2_bzReadOpen(&bz2err, f, 0, 0, NULL, 0))) + errx(1, "BZ2_bzReadOpen, bz2err=%d", bz2err); + + stream.read = bz2_read; + stream.opaque = bz2; + if (bspatch(old, oldsize, new, newsize, &stream)) + errx(1, "bspatch"); + + /* Clean up the bzip2 reads */ + BZ2_bzReadClose(&bz2err, bz2); + fclose(f); + + /* Write the new file */ + if(((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) || + (write(fd,new,newsize)!=newsize) || (close(fd)==-1)) + err(1,"%s",argv[2]); + + free(new); + free(old); + + return 0; +} + +#endif +#endif diff --git a/pkg/osvscanner/osvscanner_internal_test.go b/pkg/osvscanner/osvscanner_internal_test.go index 2986aff337..95985d1d59 100644 --- a/pkg/osvscanner/osvscanner_internal_test.go +++ b/pkg/osvscanner/osvscanner_internal_test.go @@ -117,8 +117,8 @@ func Test_scanGit(t *testing.T) { if (err != nil) != tt.wantErr { t.Errorf("scanGit() error = %v, wantErr %v", err, tt.wantErr) } - if diff := cmp.Diff(tt.wantPkg, pkg); diff != "" { - t.Errorf("scanGit() package = %v, wantPackage %v", pkg, tt.wantPkg) + if diff := cmp.Diff(pkg, tt.wantPkg); diff != "" { + t.Errorf("scanGit() returned unexpected result (-got +want):\n%s", diff) } } diff --git a/pkg/osvscanner/vendored_libs_test.go b/pkg/osvscanner/vendored_libs_test.go new file mode 100644 index 0000000000..1eff70b917 --- /dev/null +++ b/pkg/osvscanner/vendored_libs_test.go @@ -0,0 +1,52 @@ +package osvscanner + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/pkg/reporter" +) + +func Test_scanDirWithVendoredLibs(t *testing.T) { + type args struct { + r reporter.Reporter + path string + } + tests := []struct { + name string + args args + want []scannedPackage + wantErr bool + }{ + { + name: "Scan bsdiff", + args: args{ + r: &reporter.VoidReporter{}, + path: "./fixtures/example-vendor/", + }, + want: []scannedPackage{ + { + Commit: "ce07a29894acd74c52b975a42c02f11d9483566a", + Source: models.SourceInfo{ + Type: "git", + Path: "fixtures/example-vendor/bsdiff", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := scanDirWithVendoredLibs(tt.args.r, tt.args.path) + if (err != nil) != tt.wantErr { + t.Errorf("scanDirWithVendoredLibs() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("scanDirWithVendoredLibs() returned unexpected result (-got +want):\n%s", diff) + } + }) + } +} From b8000a7518bf59f9817d6fe63a338b44ccf1f3c0 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 1 Dec 2023 14:33:13 +1100 Subject: [PATCH 3/4] Fix lints --- pkg/osvscanner/osvscanner.go | 2 +- pkg/osvscanner/vendored_libs.go | 1 + pkg/osvscanner/vendored_libs_test.go | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 90faa85206..5b91d10e05 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -1,7 +1,7 @@ package osvscanner import ( - "bufio" //nolint:gosec + "bufio" "errors" "fmt" "os" diff --git a/pkg/osvscanner/vendored_libs.go b/pkg/osvscanner/vendored_libs.go index 51f67fcac6..d7c360dbc5 100644 --- a/pkg/osvscanner/vendored_libs.go +++ b/pkg/osvscanner/vendored_libs.go @@ -2,6 +2,7 @@ package osvscanner import ( "bytes" + //nolint:gosec "crypto/md5" "errors" "fmt" diff --git a/pkg/osvscanner/vendored_libs_test.go b/pkg/osvscanner/vendored_libs_test.go index 1eff70b917..6cdcc7c155 100644 --- a/pkg/osvscanner/vendored_libs_test.go +++ b/pkg/osvscanner/vendored_libs_test.go @@ -9,6 +9,8 @@ import ( ) func Test_scanDirWithVendoredLibs(t *testing.T) { + t.Parallel() + type args struct { r reporter.Reporter path string @@ -37,7 +39,9 @@ func Test_scanDirWithVendoredLibs(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() got, err := scanDirWithVendoredLibs(tt.args.r, tt.args.path) if (err != nil) != tt.wantErr { t.Errorf("scanDirWithVendoredLibs() error = %v, wantErr %v", err, tt.wantErr) From 33f20cd581dbf55f74d174ba9e3ef57da4bc4784 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 8 Dec 2023 15:33:31 +1100 Subject: [PATCH 4/4] Fix test on windows --- pkg/osvscanner/vendored_libs_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/osvscanner/vendored_libs_test.go b/pkg/osvscanner/vendored_libs_test.go index 6cdcc7c155..2e99a8a18c 100644 --- a/pkg/osvscanner/vendored_libs_test.go +++ b/pkg/osvscanner/vendored_libs_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv-scanner/internal/testutility" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/reporter" ) @@ -32,7 +33,9 @@ func Test_scanDirWithVendoredLibs(t *testing.T) { Commit: "ce07a29894acd74c52b975a42c02f11d9483566a", Source: models.SourceInfo{ Type: "git", - Path: "fixtures/example-vendor/bsdiff", + Path: testutility.ValueIfOnWindows( + "fixtures\\example-vendor\\bsdiff", + "fixtures/example-vendor/bsdiff"), }, }, },