MTC Enhancements March 2020

__tests__/test-utils/mock-data/manager.js

@@ -139,6 +139,7 @@ export const mockFeedWithVersion = {
   retrievalMethod: 'MANUALLY_UPLOADED',
   s3Url: null,
   snapshotVersion: null,
+  transformRules: [],
   url: 'http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip',
   user: null
 }
@@ -160,6 +161,7 @@ export const mockFeedWithoutVersion = {
   retrievalMethod: 'FETCHED_AUTOMATICALLY',
   s3Url: null,
   snapshotVersion: null,
+  transformRules: [],
   url: null,
   user: null
 }

gtfs.yml

@@ -776,6 +776,7 @@
 
 - id: scheduleexception
   name: (none)
+  datatools: true
   helpContent: Conveyal-specific table for classifying schedule exceptions.
   fields:
   - name: name

i18n/english.yml

@@ -228,6 +228,23 @@ components:
     upload: Upload
     versions: Versions
     viewPublic: View public page
+  FeedTransformationDescriptions:
+    general:
+      fileDefined: below text
+      filePlaceholder: '[choose file]'
+      tablePlaceholder: '[choose table]'
+      table: table
+      version: version
+      versionPlaceholder: '[choose version]'
+    DeleteRecordsTransformation:
+      label: Delete records from %tablePlaceholder%
+      name: Delete records transformation
+    ReplaceFileFromStringTransformation:
+      label: Replace %tablePlaceholder% from %filePlaceholder%
+      name: Replace file from string transformation
+    ReplaceFileFromVersionTransformation:
+      label: Replace %tablePlaceholder% from %versionPlaceholder%
+      name: Replace file from version transformation
   FeedVersionNavigator:
     confirmDelete: Are you sure you want to delete this version? This cannot be undone.
     confirmLoad: 'This will override all active GTFS Editor data for this Feed Source with the data from this version. If there is unsaved work in the Editor you want to keep, you must snapshot the current Editor data first. Are you sure you want to continue?'

lib/common/actions/index.js

@@ -11,7 +11,7 @@ export function createVoidPayloadAction (type: string) {
   return () => ({ type })
 }
 
-export function secureFetch (url: string, method: string = 'get', payload: any, raw: boolean = false, isJSON: boolean = true, actionOnFail: any): any {
+export function secureFetch (url: string, method: string = 'get', payload?: any, raw: boolean = false, isJSON: boolean = true, actionOnFail?: string): any {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     function consoleError (message) {
       console.error(`Error making ${method} request to ${url}: `, message)
@@ -95,7 +95,7 @@ export function fetchGraphQL ({
 }: {
   errorMessage?: string,
   query: string,
-  variables: any
+  variables?: {[key: string]: string | number | Array<string>}
 }): any {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const body = {

lib/common/constants/index.js

@@ -1,4 +1,5 @@
 // @flow
+
 const SECURE: string = 'secure/'
 export const API_PREFIX: string = `/api/manager/`
 export const SECURE_API_PREFIX: string = `${API_PREFIX}${SECURE}`
@@ -9,3 +10,18 @@ export const DEFAULT_DESCRIPTION = 'A command center for managing, editing, vali
 export const DEFAULT_LOGO = 'https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png'
 export const DEFAULT_LOGO_SMALL = 'https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png'
 export const DEFAULT_TITLE = 'Data Tools'
+
+export const RETRIEVAL_METHODS = Object.freeze({
+  MANUALLY_UPLOADED: 'MANUALLY_UPLOADED',
+  FETCHED_AUTOMATICALLY: 'FETCHED_AUTOMATICALLY',
+  PRODUCED_IN_HOUSE: 'PRODUCED_IN_HOUSE',
+  SERVICE_PERIOD_MERGE: 'SERVICE_PERIOD_MERGE',
+  REGIONAL_MERGE: 'REGIONAL_MERGE',
+  VERSION_CLONE: 'VERSION_CLONE'
+})
+
+export const FEED_TRANSFORMATION_TYPES = Object.freeze({
+  DELETE_RECORDS: 'DeleteRecordsTransformation',
+  REPLACE_FILE_FROM_VERSION: 'ReplaceFileFromVersionTransformation',
+  REPLACE_FILE_FROM_STRING: 'ReplaceFileFromStringTransformation'
+})

lib/common/util/config.js

@@ -37,7 +37,7 @@ export function getGtfsPlusSpec (): Array<GtfsSpecTable> {
   const CONFIG: DataToolsConfig = window.DT_CONFIG
   const GTFS_PLUS_SPEC = CONFIG.specifications.gtfsplus
   if (!GTFS_PLUS_SPEC) throw new Error('GTFS+ yml configuration file is not defined!')
-  return GTFS_PLUS_SPEC
+  return GTFS_PLUS_SPEC.sort((table1, table2) => table1.name.localeCompare(table2.name))
 }
 
 /**
@@ -58,7 +58,7 @@ export function getComponentMessages (
       return message
     } else {
       if (logWarning) console.warn(`Couldn't find message entry for ${componentName}.${path}`)
-      return `{${path}}`
+      return message || `{${path}}`
     }
   }
 }

lib/editor/actions/trip.js

@@ -92,6 +92,7 @@ export function fetchTripsForCalendar (
 ) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String, $pattern_id: [String], $service_id: [String]) {
@@ -277,6 +278,7 @@ export function fetchCalendarTripCountsForPattern (
 ) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String, $pattern_id: String) {
@@ -304,6 +306,7 @@ export function fetchCalendarTripCountsForPattern (
 export function fetchTripCounts (feedId: string) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String) {

lib/gtfsplus/components/GtfsPlusVersionSummary.js

@@ -1,14 +1,24 @@
 // @flow
 
+import moment from 'moment'
 import React, {Component} from 'react'
-import {Panel, Row, Col, Table, Button, ButtonToolbar, Glyphicon, Alert} from 'react-bootstrap'
+import {
+  Alert,
+  Button,
+  ButtonToolbar,
+  Col,
+  Glyphicon,
+  Panel,
+  Row,
+  Table
+} from 'react-bootstrap'
 import {browserHistory, Link} from 'react-router'
-import moment from 'moment'
 
-import {getGtfsPlusSpec} from '../../common/util/config'
 import * as gtfsPlusActions from '../actions/gtfsplus'
 
+import {getGtfsPlusSpec} from '../../common/util/config'
 import type {Props as ContainerProps} from '../containers/ActiveGtfsPlusVersionSummary'
+import type {GtfsPlusValidationIssue} from '../../types'
 import type {GtfsPlusReducerState, ManagerUserState} from '../../types/reducers'
 
 type Props = ContainerProps & {
@@ -51,7 +61,7 @@ export default class GtfsPlusVersionSummary extends Component<Props, State> {
     return issuesForTable[tableId].length.toLocaleString()
   }
 
-  _getTableLevelIssues = (tableId: string) => {
+  _getTableLevelIssues = (tableId: string): ?Array<GtfsPlusValidationIssue> => {
     const {issuesForTable} = this.props
     if (!issuesForTable) return null
     if (!(tableId in issuesForTable)) return null
@@ -214,7 +224,7 @@ export default class GtfsPlusVersionSummary extends Component<Props, State> {
                               {tableLevelIssues.length} critical table issue(s):
                               <ul>
                                 {tableLevelIssues.map((issue, i) =>
-                                  <li key={i}>{issue.description}</li>)}
+                                  <li key={i}>{issue.fieldName}: {issue.description}</li>)}
                               </ul>
                             </small>
                             : null

lib/manager/actions/__tests__/__snapshots__/projects.js.snap

@@ -67,6 +67,7 @@ Object {
                 "retrievalMethod": "MANUALLY_UPLOADED",
                 "s3Url": null,
                 "snapshotVersion": null,
+                "transformRules": Array [],
                 "url": "http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip",
                 "user": null,
               },
@@ -162,6 +163,7 @@ Object {
           "retrievalMethod": "MANUALLY_UPLOADED",
           "s3Url": null,
           "snapshotVersion": null,
+          "transformRules": Array [],
           "url": "http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip",
           "user": null,
         },

lib/manager/actions/status.js

@@ -8,10 +8,10 @@ import {API_PREFIX} from '../../common/constants'
 import {isExtensionEnabled} from '../../common/util/config'
 import { fetchDeployment } from './deployments'
 import { fetchFeedSource } from './feeds'
-import { downloadMergedFeedViaToken } from './projects'
+import { fetchProjectWithFeeds } from './projects'
 import { downloadSnapshotViaCredentials } from '../../editor/actions/snapshots'
 
-import type {DataToolsConfig, ServerJob} from '../../types'
+import type {DataToolsConfig, MergeFeedsResult, ServerJob} from '../../types'
 import type {dispatchFn, getStateFn} from '../../types/reducers'
 
 type ErrorMessage = {
@@ -21,6 +21,13 @@ type ErrorMessage = {
   title?: string
 }
 
+type ModalContent = {
+  action?: any,
+  body: string,
+  detail?: any,
+  title: string
+}
+
 export const clearStatusModal = createVoidPayloadAction('CLEAR_STATUS_MODAL')
 const handlingFinishedJob = createAction(
   'HANDLING_FINISHED_JOB',
@@ -57,12 +64,7 @@ const setAppInfo = createAction(
 
 const setStatusModal = createAction(
   'SET_STATUS_MODAL',
-  (payload: {
-    action?: any,
-    body: string,
-    detail?: any,
-    title: string
-  }) => payload
+  (payload: ModalContent) => payload
 )
 
 export type StatusActions = ActionType<typeof clearStatusModal> |
@@ -134,6 +136,48 @@ export async function fetchAppInfo () {
   }
 }
 
+function getMergeFeedModalContent (result: MergeFeedsResult): ModalContent {
+  const details = []
+  // Do nothing or show merged feed modal? Feed version is be created
+  details.push('Remapped ID count: ' + result.remappedReferences)
+  if (Object.keys(result.remappedIds).length > 0) {
+    const remappedIdStrings = []
+    for (let key in result.remappedIds) {
+      // Modify key to remove feed name.
+      const split = key.split(':')
+      const tableAndId = split.splice(1, 1)
+      remappedIdStrings.push(`${tableAndId.join(':')} -> ${result.remappedIds[key]}`)
+    }
+    details.push('Remapped IDs: ' + remappedIdStrings.join(', '))
+  }
+  if (result.skippedIds.length > 0) {
+    const skippedRecordsForTables = {}
+    result.skippedIds.forEach(id => {
+      const table = id.split(':')[0]
+      // Increment count of skipped records for each value found per table.
+      skippedRecordsForTables[table] = (skippedRecordsForTables[table] || 0) + 1
+    })
+    const skippedRecordsStrings = []
+    for (let key in skippedRecordsForTables) {
+      skippedRecordsStrings.push(`${key} - ${skippedRecordsForTables[key]}`)
+    }
+    details.push('Skipped records: ' + skippedRecordsStrings.join(', '))
+  }
+  if (result.idConflicts.length > 0) {
+    // const conflicts = result.idConflicts
+    details.push('ID conflicts: ' + result.idConflicts.join(', '))
+  }
+  return {
+    title: result.failed
+      ? 'Warning: Errors encountered during feed merge!'
+      : 'Feed merge was successful!',
+    body: result.failed
+      ? `Merge failed with ${result.errorCount} errors. ${result.failureReasons.join(', ')}`
+      : `Merge was completed successfully. A new version will be processed/validated containing the resulting feed.`,
+    detail: details.join('\n')
+  }
+}
+
 /* eslint-disable complexity */
 export function handleFinishedJob (job: ServerJob) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
@@ -203,60 +247,21 @@ export function handleFinishedJob (job: ServerJob) {
           // If merging feeds for the project, end result is to download zip.
           if (!job.projectId) {
             // FIXME use setErrorMessage instead?
-            console.warn('No project found on job')
-            return
+            throw new Error('No project found on job')
           }
-          dispatch(downloadMergedFeedViaToken(job.projectId, false))
+          // FIXME
+          dispatch(fetchProjectWithFeeds(job.projectId))
         } else {
           const result = job.mergeFeedsResult
-          const details = []
           if (result) {
-            // Do nothing or show merged feed modal? Feed version is be created
-            details.push('Remapped ID count: ' + result.remappedReferences)
-            if (Object.keys(result.remappedIds).length > 0) {
-              const remappedIdStrings = []
-              for (let key in result.remappedIds) {
-                // Modify key to remove feed name.
-                const split = key.split(':')
-                const tableAndId = split.splice(1, 1)
-                remappedIdStrings.push(`${tableAndId.join(':')} -> ${result.remappedIds[key]}`)
-              }
-              details.push('Remapped IDs: ' + remappedIdStrings.join(', '))
-            }
-            if (result.skippedIds.length > 0) {
-              const skippedRecordsForTables = {}
-              result.skippedIds.forEach(id => {
-                const table = id.split(':')[0]
-                if (skippedRecordsForTables[table]) {
-                  skippedRecordsForTables[table] = skippedRecordsForTables[table] + 1
-                } else {
-                  skippedRecordsForTables[table] = 1
-                }
-              })
-              const skippedRecordsStrings = []
-              for (let key in skippedRecordsForTables) {
-                skippedRecordsStrings.push(`${key} - ${skippedRecordsForTables[key]}`)
-              }
-              details.push('Skipped records: ' + skippedRecordsStrings.join(', '))
-            }
-            if (result.idConflicts.length > 0) {
-              // const conflicts = result.idConflicts
-              details.push('ID conflicts: ' + result.idConflicts.join(', '))
-            }
-            dispatch(setStatusModal({
-              title: result.failed
-                ? 'Warning: Errors encountered during feed merge!'
-                : 'Feed merge was successful!',
-              body: result.failed
-                ? `Merge failed with ${result.errorCount} errors. ${result.failureReasons.join(', ')}`
-                : `Merge was completed successfully. A new version will be processed/validated containing the resulting feed.`,
-              detail: details.join('\n')
-            }))
+            const modalContent = getMergeFeedModalContent(result)
+            dispatch(setStatusModal(modalContent))
           }
         }
         break
       default:
         console.warn(`No completion step defined for job type ${job.type}`)
+        break
     }
   }
 }

lib/manager/actions/versions.js

@@ -164,7 +164,7 @@ export function publishFeedVersion (feedVersion: FeedVersion) {
 /**
  * Merges two feed versions according to the strategy defined within the
  */
-export function mergeVersions (targetVersionId: string, versionId: string, mergeType: string) {
+export function mergeVersions (targetVersionId: string, versionId: string, mergeType: 'SERVICE_PERIOD' | 'REGIONAL') {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const url = `${SECURE_API_PREFIX}feedversion/merge?feedVersionIds=${targetVersionId},${versionId}&mergeType=${mergeType}`
     return dispatch(secureFetch(url, 'put'))
@@ -290,11 +290,11 @@ export function fetchGTFSEntities ({
         }
       }
     `
-    // If fetching for the editor, cast id to int for csv_line field
-    return dispatch(fetchGraphQL({
-      query,
-      variables: {namespace, [entityIdField]: editor ? +id : id}
-    }))
+    const variables = !id
+      ? {namespace}
+      // If fetching a single ID for the editor, cast id to int for csv_line field
+      : {namespace, [entityIdField]: editor ? +id : id}
+    return dispatch(fetchGraphQL({query, variables}))
       .then(data => {
         dispatch(receiveGTFSEntities({namespace, id, component: type, data, editor, replaceNew}))
         if (editor) {