Merge pull request #7 from ministryofjustice/local-artificial-branches

Local artificial branches

.dockerignore

@@ -1,5 +1,6 @@
 .git/
 .github/
+output/
 .DS_Store
 .gitignore
 LICENSE

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
 *.sql
+output/

README.md

@@ -1,12 +1,12 @@
 ```
-  ______            _   _____                         
- |  ____|          | | |  __ \                        
- | |__ ___  ___  __| | | |__) |_ _ _ __ ___  ___ _ __ 
+  ______            _   _____
+ |  ____|          | | |  __ \
+ | |__ ___  ___  __| | | |__) |_ _ _ __ ___  ___ _ __
  |  __/ _ \/ _ \/ _` | |  ___/ _` | '__/ __|/ _ \ '__|
- | | |  __/  __/ (_| | | |  | (_| | |  \__ \  __/ |   
- |_|  \___|\___|\__,_| |_|   \__,_|_|  |___/\___|_|   
+ | | |  __/  __/ (_| | | |  | (_| | |  \__ \  __/ |
+ |_|  \___|\___|\__,_| |_|   \__,_|_|  |___/\___|_|
 
-```                                                     
+```
 [![Feed Parser Deployment](https://github.com/ministryofjustice/feed-parser/actions/workflows/cd.yaml/badge.svg)](https://github.com/ministryofjustice/feed-parser/actions/workflows/cd.yaml)
 
 # FeedParser
@@ -24,3 +24,14 @@ To check if the image has been pushed into the ECR repo make sure in your termin
 ## Usage
 
 We are currently running this application in [kubernetes using a cron manifest file](https://github.com/ministryofjustice/hale-platform/blob/main/helm_deploy/wordpress/templates/cron-feedparser.yaml) that periodically runs depending on the cron schedule.
+
+## Local development
+
+To run locally:
+
+1. In this repo root directory run `make build`.
+2. Run `make run`
+
+To stop run `make down` .
+
+Files will be exported to the `/output` folder rather then exported to s3.

docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+  feedparser-local:
+    image: feedparser-local
+    volumes:
+      - ./output:/output
+    container_name: feedparser-local
+    environment:
+      - ENV_TYPE=local
+    command: []

inc/OleeoFeedParser.php

@@ -372,7 +372,7 @@ function validateOptionalFieldsbySpan($job, $jobContent){
         ];
 
         foreach ($artificial_role_types as $job_type) {
-            if(str_contains($job['title'], $job_type)) {
+            if(strpos("x".$job['title'], $job_type)) {
                 array_push($job['roleTypes'], (string) $job_type);
                 continue;
             }
@@ -399,7 +399,9 @@ function validateOptionalFieldsbyNewLine($job, $jobContent){
         }
 
         foreach ($artificial_role_types as $job_type) {
-            if(str_contains($job['title'], $job_type)) {
+            if(strpos("x".$job['title'], $job_type)) {
+                // strpos returns false if the "needle" is at position zero in the "haystack",
+                // so we add a character to the beginning to ensure that's not going to happen.
                 array_push($job['roleTypes'], (string) $job_type);
                 continue;
             }

makefile

@@ -0,0 +1,11 @@
+build:
+	docker build -f dockerfile -t feedparser-local .
+
+run:
+	docker compose up -d
+
+down:
+	docker compose down
+
+log:
+	docker logs feedparser-local

run.php

@@ -7,6 +7,32 @@
 use Aws\S3\S3Client;
 use Aws\Exception\AwsException;
 
+// Check environment
+$envType = getenv('ENV_TYPE');
+
+if ($envType === 'local') {
+    echo 'The environment type is local export files locally.';
+}
+
+if ($envType != 'local') {
+    echo 'The environment type is not local, setup and push to s3.';
+
+     // AWS S3 Bucket Name
+    $s3BucketName = getenv('S3_UPLOADS_BUCKET');
+
+    // AWS S3 Bucket Path
+    $s3BucketPath = 'feed-parser/';  // Set to an empty string if the root of the bucket
+
+    // AWS S3 Region
+    $awsRegion = 'eu-west-2';
+
+    // Create an S3Client with the AWS credentials automatically provided by IAM instance profile
+    $s3Client = new S3Client([
+        'region' => $awsRegion,
+        'version' => 'latest'
+    ]);
+}
+
 echo 'Feed Parser Started';
 
 $feeds = [
@@ -42,26 +68,12 @@
  ];
 
 if (count($feeds) == 0) {
-    return;
+    exit;
 }
 
 $availableFeeds = [];
 
- // AWS S3 Bucket Name
-$s3BucketName = getenv('S3_UPLOADS_BUCKET');
-
-// AWS S3 Bucket Path
-$s3BucketPath = 'feed-parser/';  // Set to an empty string if the root of the bucket
-
-// AWS S3 Region
-$awsRegion = 'eu-west-2';
-
-// Create an S3Client with the AWS credentials automatically provided by IAM instance profile
-$s3Client = new S3Client([
-    'region' => $awsRegion,
-    'version' => 'latest'
-]);
-
+$uploadResult = [];
 
 foreach ($feeds as $feed) {
     $feedID = $feed['id'];
@@ -89,7 +101,7 @@
 
     $filters = [];
 
-    if(array_key_exists('filters', $feed) && !empty($feed['filters'])){
+    if (array_key_exists('filters', $feed) && !empty($feed['filters'])) {
         $filters = $feed['filters'];
     }
 
@@ -101,31 +113,54 @@
         continue;
     }
 
-    $uploadResult = uploadFiletoS3($s3Client, $s3BucketName, $s3BucketPath . "$feedID.json", $jsonFile);
-
-    if (!$uploadResult['success'] || empty($uploadResult['fileURL'])) {
-        continue;
+    if ($envType === 'local') {
+        $uploadResult['fileURL'] = $jsonFile;
     }
 
     $availableFeeds[] = [
         'name' => $feed['name'],
-        'url' => $uploadResult['fileURL']
+        'url' => isset($uploadResult['fileURL']) ? $uploadResult['fileURL'] : null,
     ];
-
+
+    // Export locally
+    if ($envType === 'local') {
+        $file_path = "output/$feedID.json";
+
+        // Use file_put_contents to save the content to the local file
+        $result = file_put_contents($file_path, file_get_contents($file_path));
+
+        if ($result !== false) {
+            echo "Feed data successfully written to the file. $result bytes written.";
+        } else {
+            echo "Error writing data to local file.";
+        }
+    }
+
+    if ($envType !== 'local') {
+        // Export to s3
+        $uploadResult = uploadFiletoS3($s3Client, $s3BucketName, $s3BucketPath . "$feedID.json", $jsonFile);
+
+        if (!$uploadResult['success'] || empty($uploadResult['fileURL'])) {
+            continue;
+        }
+    }
 }
 
 //Create Available Feeds JSON File
-
 $feedsJSON = json_encode($availableFeeds);
 
 if (!$feedsJSON) {
-    return;
+    exit;
 }
 
 $writeFileResult = file_put_contents("output/feeds.json", $feedsJSON);
 
 if ($writeFileResult === false) {
-    return;
+    exit;
+}
+
+if ($envType == 'local') {
+    exit;
 }
 
 $result = uploadFiletoS3($s3Client, $s3BucketName, $s3BucketPath . "feeds.json", "output/feeds.json");
@@ -134,7 +169,7 @@ function uploadFiletoS3($s3Client, $s3BucketName, $s3ObjectKey, $sourceFile)
 {
 
     $uploadResult = [
-        "success" => false, 
+        "success" => false,
         "fileURL" => false
     ];
 
@@ -157,9 +192,8 @@ function uploadFiletoS3($s3Client, $s3BucketName, $s3ObjectKey, $sourceFile)
         echo 'Error: ' . $e->getMessage() . PHP_EOL;
     }
 
-    if(!empty($result) && array_key_exists('effectiveUri', $result['@metadata'])){
-
-        $uploadResult['fileURL'] = $result['@metadata']['effectiveUri'];      
+    if (!empty($result) && array_key_exists('effectiveUri', $result['@metadata'])) {
+        $uploadResult['fileURL'] = $result['@metadata']['effectiveUri'];
     }
 
     return $uploadResult;