From 5fb9ccf54aefc448dfe87abed3a77105b5a0738e Mon Sep 17 00:00:00 2001 From: Seamus Abshere Date: Fri, 10 Feb 2012 19:20:24 -0600 Subject: [PATCH] first commit --- .gitignore | 17 ++ Gemfile | 7 + README.markdown | 22 ++ Rakefile | 12 ++ lib/unix_utils.rb | 209 +++++++++++++++++++ lib/unix_utils/version.rb | 3 + test/.DS_Store | Bin 0 -> 6148 bytes test/helper.rb | 75 +++++++ test/ready_made/directory-really-a-t_a_r-shh | Bin 0 -> 3584 bytes test/ready_made/directory-really-a-z_i_p-shh | Bin 0 -> 369 bytes test/ready_made/directory.tar | Bin 0 -> 3584 bytes test/ready_made/directory.zip | Bin 0 -> 369 bytes test/ready_made/file-really-a-b_z_2-shh | Bin 0 -> 52 bytes test/ready_made/file-really-a-g_z-shh | Bin 0 -> 48 bytes test/ready_made/file.bz2 | Bin 0 -> 52 bytes test/ready_made/file.gz | Bin 0 -> 48 bytes test/test_unix_utils.rb | 152 ++++++++++++++ test/to_be_processed/hello_world.txt | 1 + test/to_be_processed/hello_world.xml | 1 + unix_utils.gemspec | 17 ++ 20 files changed, 516 insertions(+) create mode 100644 .gitignore create mode 100644 Gemfile create mode 100644 README.markdown create mode 100644 Rakefile create mode 100644 lib/unix_utils.rb create mode 100644 lib/unix_utils/version.rb create mode 100644 test/.DS_Store create mode 100644 test/helper.rb create mode 100644 test/ready_made/directory-really-a-t_a_r-shh create mode 100644 test/ready_made/directory-really-a-z_i_p-shh create mode 100644 test/ready_made/directory.tar create mode 100644 test/ready_made/directory.zip create mode 100644 test/ready_made/file-really-a-b_z_2-shh create mode 100644 test/ready_made/file-really-a-g_z-shh create mode 100644 test/ready_made/file.bz2 create mode 100644 test/ready_made/file.gz create mode 100644 test/test_unix_utils.rb create mode 100644 test/to_be_processed/hello_world.txt create mode 100644 test/to_be_processed/hello_world.xml create mode 100644 unix_utils.gemspec diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d87d4be --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..7dde22b --- /dev/null +++ b/Gemfile @@ -0,0 +1,7 @@ +source :rubygems + +gemspec + +# development dependencies +gem 'minitest' +gem 'minitest-reporters' \ No newline at end of file diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000..5198603 --- /dev/null +++ b/README.markdown @@ -0,0 +1,22 @@ +# unix_utils + +Dead simple access to bzip2, bunzip2, tar, untar, du, sha256, etc. + +TODO: md5, wc, etc. + +## Philosophy + +* Give a path, get a path (bzip2, bunzip2, etc.) +* Only return the good part (sha256, du, etc.) + +## Spawning + +Uses `open3` because it's the most consistent interface across MRI 1.8, 1.9 and JRuby. + +## Authors + +* Seamus Abshere + +## Copyright + +Copyright (c) 2012 Brighter Planet. See LICENSE for details. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..e231eaf --- /dev/null +++ b/Rakefile @@ -0,0 +1,12 @@ +#!/usr/bin/env rake +require "bundler/gem_tasks" + +require 'rake' +require 'rake/testtask' +Rake::TestTask.new(:test) do |test| + test.libs << 'lib' << 'test' + test.pattern = 'test/**/test_*.rb' + test.verbose = true +end + +task :default => :test diff --git a/lib/unix_utils.rb b/lib/unix_utils.rb new file mode 100644 index 0000000..b0f4941 --- /dev/null +++ b/lib/unix_utils.rb @@ -0,0 +1,209 @@ +require 'fileutils' +require 'tmpdir' +require 'uri' +require 'open3' + +require "unix_utils/version" + +module UnixUtils + + def self.curl(url, form_data = nil) + out_path = tmp_path url + if url.start_with?('/') or url.start_with?('file://') + # deal with local files + ::FileUtils.cp url.sub('file://', ''), path + return out_path + end + uri = ::URI.parse url + argv = [ 'curl', '--location', '--show-error', '--silent', '--compressed', '--header', 'Expect: ' ] + if form_data + argv += [ '--data', form_data ] + end + argv += [ uri.to_s, '--output', out_path ] + spawn *argv + out_path + end + + def self.sha256(in_path) + stdout = spawn 'shasum', '-a', '256', :stdin => in_path + stdout.chomp.split(/\s+/).first + end + + def self.du(in_dir) + stdout = spawn 'du', in_dir + stdout.chomp.split(/\s+/).first.to_i + end + + # -- + + def self.unzip(in_path) + out_dir = tmp_path in_path + ::FileUtils.mkdir out_dir + spawn 'unzip', '-qq', '-n', in_path, '-d', out_dir + out_dir + end + + def self.untar(in_path) + out_dir = tmp_path in_path + ::FileUtils.mkdir out_dir + spawn 'tar', '-xf', in_path, '-C', out_dir + out_dir + end + + def self.gunzip(in_path) + out_path = tmp_path in_path + spawn 'gunzip', '--stdout', in_path, :stdout => out_path + out_path + end + + def self.bunzip2(in_path) + out_path = tmp_path in_path + spawn 'bunzip2', '--stdout', in_path, :stdout => out_path + out_path + end + + # -- + + def self.bzip2(in_path) + out_path = tmp_path in_path + spawn 'bzip2', '--keep', '--stdout', in_path, :stdout => out_path + out_path + end + + def self.tar(in_dir) + out_path = tmp_path in_dir + spawn 'tar', '-cf', out_path, '-C', in_dir, '.' + out_path + end + + def self.zip(in_dir) + out_path = tmp_path in_dir, 'zip' + spawn 'zip', '-rq', out_path, '.', :chdir => in_dir + out_path + end + + def self.gzip(in_path) + out_path = tmp_path in_path + spawn 'gzip', '--stdout', in_path, :stdout => out_path + out_path + end + # + # # use awk to convert [CR]LF to CRLF + # def self.unix2dos(path) + # out_path = tmp_path + # ::File.open(out_path, 'wb') do |out| + # see also the simpler version + # pid = ::POSIX::Spawn.spawn 'awk', '{ sub(/\r?$/,"\r"); print }', path, :out => out + # ::Process.waitpid pid + # end + # ::FileUtils.mv out_path, path + # path + # end + + + # + # + # + # + # 'wc', '-l', in_path).split(/\s+/)[0].try :to_i + # + # sha1sum + # bang('sha1sum', in_path)[0..39] + # + # + # def self.in_place(*args) + # options = args.extract_options! + # in_path = args.shift + # argv = args + # out_path = tmp_path in_path + # ::File.open(in_path, 'r') do |f0| + # ::File.open(out_path, 'wb') do |f1| + # spawn *argv, :in => f0, :out => f1 + # end + # end + # ::FileUtils.mv out_path, in_path + # nil + # rescue SpawnError => e + # if options[:ignore_error] + # $stderr.puts "#{e.inspect} (ignoring error...)" + # ::FileUtils.mv out_path, in_path + # else + # raise e + # end + # end + # # + # # + # gsed "s/#{quoted_clone_name.gsub('`', %{\\\\`})}/#{resource_model.quoted_table_name.gsub('`', %{\\\\`})}/g" --in-place="" #{p}} + # + # # use awk to convert [CR]LF to CRLF + # def self.unix2dos(in_path) + # in_place 'awk', '{ sub(/\r?$/,"\r"); print }', in_path + # end + + # mostly for internal use... + + def self.tmp_path(ancestor, extname = nil) + basename = ::File.basename ancestor.gsub(/\W+/, '_') + basename.sub!(/^unix_utils-[0-9]+-/, '') + if extname + basename.concat ".#{extname}" + end + ::Kernel.srand + ::File.join ::Dir.tmpdir, "unix_utils-#{::Kernel.rand(1e11)}-#{basename}" + end + + def self.spawn(*argv) + argv = argv.dup + options = argv.last.is_a?(::Hash) ? argv.pop : {} + + if options[:chdir] + old_pwd = ::Dir.pwd + ::Dir.chdir options[:chdir] + end + + whole_stdout = nil + whole_stderr = nil + + if options[:stdin] + ::File.open(options[:stdin], 'r') do |in_f| + ::Open3.popen3(*argv) do |stdin, stdout, stderr| + while chunk = in_f.read(4_194_304) + stdin.write chunk + end + stdin.close + whole_stdout = stdout.read + whole_stderr = stderr.read + end + end + elsif options[:stdout] + ::File.open(options[:stdout], 'wb') do |out_f| + ::Open3.popen3(*argv) do |stdin, stdout, stderr| + stdin.close + while chunk = stdout.read(4_194_304) + out_f.write chunk + end + whole_stdout = "Redirected to #{options[:stdout]}" + whole_stderr = stderr.read + end + end + else + ::Open3.popen3(*argv) do |stdin, stdout, stderr| + stdin.close + whole_stdout = stdout.read + whole_stderr = stderr.read + end + end + + unless whole_stderr.empty? + $stderr.puts "[unix_utils] `#{argv.join(' ')}` STDERR:" + $stderr.puts whole_stderr + end + + whole_stdout + + ensure + if options[:chdir] + ::Dir.chdir old_pwd + end + end +end diff --git a/lib/unix_utils/version.rb b/lib/unix_utils/version.rb new file mode 100644 index 0000000..bbdb2d9 --- /dev/null +++ b/lib/unix_utils/version.rb @@ -0,0 +1,3 @@ +module UnixUtils + VERSION = "0.0.1" +end diff --git a/test/.DS_Store b/test/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0{ z=N$8p7<7eyJtt{ntnIG%uD7>jn1%p^Q4h!$zmNUr`Nn?`iWABLaM_jobN{{j)VqJ# zX%m6dEAYAhQ)>fE{B{3D_S%0Cc>h62y#I0k1DLTXSybiQ2`JL4jAvjYQG7-Pq5@HY HQ!4NP!2w&C literal 0 HcmV?d00001 diff --git a/test/ready_made/directory-really-a-z_i_p-shh b/test/ready_made/directory-really-a-z_i_p-shh new file mode 100644 index 0000000000000000000000000000000000000000..c258d8f6cc9172f9db4c52f61898a6858333f36f GIT binary patch literal 369 zcmWIWW@h1H00F%^F9%&~j`ci1HVE@G$S`E2=H%qZm**Gdr0A7Yl!S(GGB7`%VCru+ z(bT`Rf}4SnE~Ic8i=mH^tpzyNd;!;(f23&R7f5D%a^9^Ft>CnFmgie@O-1Go%jWdpf^2?$pK J>5U)`0{}6yOh^C# literal 0 HcmV?d00001 diff --git a/test/ready_made/directory.tar b/test/ready_made/directory.tar new file mode 100644 index 0000000000000000000000000000000000000000..4ddfddebd72e95fa4840d6ae2b620ad2c3455cb2 GIT binary patch literal 3584 zcmeHH!3u*g4DESe(J#=p+Vmp+%BZa3p(0pie*R*eY*S&U!f?$c^ih(R_eheI1P{iT zl9G}+!;+cE{Tm@4lyd+jflI|G{ z=N$8p7<7eyJtt{ntnIG%uD7>jn1%p^Q4h!$zmNUr`Nn?`iWABLaM_jobN{{j)VqJ# zX%m6dEAYAhQ)>fE{B{3D_S%0Cc>h62y#I0k1DLTXSybiQ2`JL4jAvjYQG7-Pq5@HY HQ!4NP!2w&C literal 0 HcmV?d00001 diff --git a/test/ready_made/directory.zip b/test/ready_made/directory.zip new file mode 100644 index 0000000000000000000000000000000000000000..c258d8f6cc9172f9db4c52f61898a6858333f36f GIT binary patch literal 369 zcmWIWW@h1H00F%^F9%&~j`ci1HVE@G$S`E2=H%qZm**Gdr0A7Yl!S(GGB7`%VCru+ z(bT`Rf}4SnE~Ic8i=mH^tpzyNd;!;(f23&R7f5D%a^9^Ft>CnFmgie@O-1Go%jWdpf^2?$pK J>5U)`0{}6yOh^C# literal 0 HcmV?d00001 diff --git a/test/ready_made/file-really-a-b_z_2-shh b/test/ready_made/file-really-a-b_z_2-shh new file mode 100644 index 0000000000000000000000000000000000000000..236db1206be6219445ea67f500562340f8e70a93 GIT binary patch literal 52 zcmZ>Y%CIzaj8qGb^n3HdjDdkEuz^9qfq~6sLW2T>A)AjQY%CIzaj8qGb^n3HdjDdkEuz^9qfq~6sLW2T>A)AjQhello world diff --git a/unix_utils.gemspec b/unix_utils.gemspec new file mode 100644 index 0000000..2166121 --- /dev/null +++ b/unix_utils.gemspec @@ -0,0 +1,17 @@ +# -*- encoding: utf-8 -*- +require File.expand_path('../lib/unix_utils/version', __FILE__) + +Gem::Specification.new do |gem| + gem.authors = ["Seamus Abshere"] + gem.email = ["seamus@abshere.net"] + gem.description = %q{TODO: Write a gem description} + gem.summary = %q{TODO: Write a gem summary} + gem.homepage = "" + + gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } + gem.files = `git ls-files`.split("\n") + gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") + gem.name = "unix_utils" + gem.require_paths = ["lib"] + gem.version = UnixUtils::VERSION +end