Add Tika Server to package

helviojunior · May 12, 2023 · b2cc7c3 · b2cc7c3
1 parent 87e887d
commit b2cc7c3
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 0 deletions.
diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml
@@ -92,6 +92,12 @@ jobs:
       - name: Checkout filecrawler
         uses: actions/checkout@v3
 
+      - name: Install dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: wget curl jq
+          version: 1.0
+
       - name: Setup Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:
@@ -102,6 +108,13 @@ jobs:
           python -m pip install --upgrade pip wheel
           python -m pip install -r requirements.txt -r tests/requirements-test.txt
 
+      - name: Get tika server .jar
+        run: |
+          VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0')
+          SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1")
+          wget -nv -O "./filecrawler/libs/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar"
+          echo "${SERVER_HASH} ./filecrawler/libs/bin/tika-server.jar" | sha1sum -c - || exit 1
+
       - name: Build artifact
         run: |
           python setup.py sdist

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -151,6 +151,12 @@ jobs:
       - name: Checkout filecrawler
         uses: actions/checkout@v3
 
+      - name: Install dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: wget curl jq
+          version: 1.0
+
       - name: Setup Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:
@@ -168,6 +174,13 @@ jobs:
           meta=$(cat filecrawler/__meta__.py | sed "s/__version__.*/__version__ = '"${oldv}"-"${current}"'/")
           echo "$meta" > filecrawler/__meta__.py
 
+      - name: Get tika server .jar
+        run: |
+          VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0')
+          SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1")
+          wget -nv -O "./filecrawler/libs/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar"
+          echo "${SERVER_HASH} ./filecrawler/libs/bin/tika-server.jar" | sha1sum -c - || exit 1
+
       - name: Install filecrawler
         run: |
           pip install .

diff --git a/filecrawler/config.py b/filecrawler/config.py
@@ -98,6 +98,8 @@ def initialize():
         # Overwrite config values with arguments (if defined)
         Configuration.load_from_arguments()
 
+        os.environ["TIKA_SERVER_JAR"] = os.path.join(Configuration.lib_path, 'tika-server.jar')
+
 
     @staticmethod
     def load_from_arguments():