-
-
Notifications
You must be signed in to change notification settings - Fork 13k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
gfal2-util, python3Packages.gfal2-util: init at 1.8.1
Add also passthru.fetchGfal2. This fetcher is currently used only by passthru.tsets, but capable to be used as a domain-specific real-world fetcher.
- Loading branch information
1 parent
30932c0
commit 7f95122
Showing
6 changed files
with
182 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{ python3Packages }: | ||
with python3Packages; toPythonApplication gfal2-util |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
{ lib | ||
, buildPythonPackage | ||
, callPackage | ||
, fetchFromGitHub | ||
, runCommandLocal | ||
# Build inputs | ||
, gfal2-python | ||
# For tests | ||
, xrootd # pkgs.xrootd | ||
}: | ||
(buildPythonPackage rec { | ||
pname = "gfal2-util"; | ||
version = "1.8.1"; | ||
src = fetchFromGitHub { | ||
owner = "cern-fts"; | ||
repo = "gfal2-util"; | ||
rev = "v${version}"; | ||
hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk="; | ||
}; | ||
|
||
# Replace the ad-hoc python executable finding | ||
# and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python` | ||
# for fixup phase to work correctly. | ||
postPatch = '' | ||
for script in src/gfal-*; do | ||
patch "$script" ${./gfal-util-script.patch} | ||
done | ||
''; | ||
|
||
propagatedBuildInputs = [ | ||
gfal2-python | ||
]; | ||
|
||
pythonImportsCheck = [ | ||
"gfal2_util" | ||
]; | ||
|
||
meta = with lib; { | ||
description = "CLI for gfal2"; | ||
homepage = "https://github.com/cern-fts/gfal2-utils"; | ||
license = licenses.asl20; | ||
maintainers = with maintainers; [ ShamrockLee ]; | ||
}; | ||
}).overrideAttrs (finalAttrs: previousAttrs: { | ||
passthru = { | ||
inherit (gfal2-python) gfal2; | ||
|
||
fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; }); | ||
|
||
# With these functionality tests, it should be safe to merge version bumps once all the tests are passed. | ||
tests = | ||
let | ||
# Use the the bin output hash of gfal2-util as version to ensure that | ||
# the test gets rebuild everytime gfal2-util gets rebuild | ||
versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}"; | ||
self = finalAttrs.finalPackage; | ||
in | ||
lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd ( | ||
let | ||
# Test against a real-world dataset from CERN Open Data | ||
# borrowed from `xrootd.tests`. | ||
urlTestFile = xrootd.tests.test-xrdcp.url; | ||
hashTestFile = xrootd.tests.test-xrdcp.outputHash; | ||
urlTestDir = dirOf urlTestFile; | ||
in | ||
{ | ||
test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 { | ||
url = urlTestFile; | ||
hash = hashTestFile; | ||
extraGfalCopyFlags = [ "--verbose" ]; | ||
pname = "gfal2-util-test-copy-file-xrootd"; | ||
version = versionFODTests; | ||
allowSubstitutes = false; | ||
}; | ||
|
||
test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 { | ||
url = urlTestDir; | ||
hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw="; | ||
recursive = true; | ||
extraGfalCopyFlags = [ "--verbose" ]; | ||
pname = "gfal2-util-test-copy-dir-xrootd"; | ||
version = versionFODTests; | ||
allowSubstitutes = false; | ||
}; | ||
|
||
test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } '' | ||
set -eu -o pipefail | ||
gfal-ls "$url" | grep "$baseNameExpected" | tee "$out" | ||
'').overrideAttrs (finalAttrs: previousAttrs: { | ||
pname = previousAttrs.name; | ||
version = versionFODTests; | ||
name = "${finalAttrs.pname}-${finalAttrs.version}"; | ||
nativeBuildInputs = [ self ]; | ||
url = urlTestDir; | ||
baseNameExpected = baseNameOf urlTestFile; | ||
outputHashMode = "flat"; | ||
outputHashAlgo = "sha256"; | ||
outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n"); | ||
}); | ||
} | ||
); | ||
}; | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
{ lib | ||
, runCommandLocal | ||
, gfal2-util | ||
}: | ||
|
||
# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`. | ||
{ name ? "" | ||
, pname ? "" | ||
, version ? "" | ||
, urls ? [ ] | ||
, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls | ||
, hash ? lib.fakeHash | ||
, recursive ? false | ||
, intermediateDestUrls ? [ ] | ||
, extraGfalCopyFlags ? [ ] | ||
, allowSubstitutes ? true | ||
}: | ||
|
||
(runCommandLocal name { } '' | ||
for u in "''${urls[@]}"; do | ||
gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out" | ||
ret="$?" | ||
(( ret )) && break | ||
done | ||
if (( ret )); then | ||
echo "gfal-copy failed trying to download from any of the urls" >&2 | ||
exit "$ret" | ||
fi | ||
'').overrideAttrs (finalAttrs: previousAttrs: | ||
{ | ||
__structuredAttrs = true; | ||
inherit allowSubstitutes; | ||
nativeBuildInputs = [ gfal2-util ]; | ||
outputHashAlgo = null; | ||
outputHashMode = if finalAttrs.recursive then "recursive" else "flat"; | ||
outputHash = hash; | ||
inherit url; | ||
urls = if urls == [ ] then lib.singleton url else urls; | ||
gfalCopyFlags = extraGfalCopyFlags | ||
++ lib.optional finalAttrs.recursive "--recursive" | ||
; | ||
inherit recursive intermediateDestUrls; | ||
} // (if (pname != "" && version != "") then { | ||
inherit pname version; | ||
name = "${finalAttrs.pname}-${finalAttrs.version}"; | ||
} else { | ||
name = if (name != "") then name else (baseNameOf url); | ||
})) |
19 changes: 19 additions & 0 deletions
19
pkgs/development/python-modules/gfal2-util/gfal-util-script.patch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
--- a/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000 | ||
+++ b/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000 | ||
@@ -1,4 +1,4 @@ | ||
-#!/bin/sh | ||
+#!/usr/bin/env python3 | ||
# -*- coding: utf-8 -*- | ||
# | ||
# Copyright (c) 2022 CERN | ||
@@ -15,10 +15,2 @@ | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
- | ||
-# Execute script content with first python interpreter found: | ||
-# * GFAL_PYTHONBIN environment variable | ||
-# * python on the PATH if import gfal2, gfal2_util succeeds | ||
-# * python3 on the PATH if import gfal2, gfal2_util succeeds | ||
-# * python2 on the PATH if import gfal2, gfal2_util succeeds | ||
-# * /usr/bin/python | ||
-"exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters