Add .add() method to ProjectSSH class (#63)

* add '.add()' method to ProjectSSH class

* update variable name in delete_file() method

* Bugfix update 3.0.4

Fixes issues in project.py:
- No .add() method when in SSH mode
- delete_file() definition had a misspelled variable

* fix spelling error in sum() definition

* bugfix updates 3.0.5

fixed a spelling error in core.py sum() definition where the measure type was 'tum'

* fix addDimension() type definition

* update dense_children_threshold to 1 for proper formatting

* Update to 3.0.6

* bugfix: keep sumAllNumDimensions from summing id fields

* Update to 3.0.7

* Update: setName_safe to add alias and not error

* update to version 3.0.8

* accepting changes in PR63

** also added sql_preamble and identified some issues with testing state

Co-authored-by: russlooker <russell@looker.com>

lookml/core.py

@@ -549,15 +549,22 @@ def setName_safe(self, newName: str):
             self for method chaining
         """
         #P2: complete checking all places for dependencies, html, links etc
-        old = copy.deepcopy(self.name)
         oldrefsre = copy.deepcopy(self.__refsre__)
         oldrefre = copy.deepcopy(self.__refre__)
+
+        # Add current name as an alias to avoid any content validation issues
+        if type(self.alias) == list:
+            self.alias.append(self.name)
+        else:
+            self.alias = [self.name]
+
+        # Change existing name to new name
         self.setName(newName)
         for f in self.parent.search('sql',[oldrefsre,oldrefre]):
-            f.sql = re.sub(oldrefsre, self.__refs__, str(f.sql.value))
-            f.sql = re.sub(oldrefre, self.__ref__, str(f.sql.value))
-        self.parent.removeField(old)
-        self.parent + self
+            f.sql = re.sub(oldrefsre, self.__refs__, f.sql.value)
+            f.sql = re.sub(oldrefre, self.__ref__, f.sql.value)
+        # self.parent.removeField(old)
+        # self.parent + self
         return self
 
     def setSql(self,sql: str):
@@ -1054,7 +1061,7 @@ def getFieldsSorted(self):
         return sorted(self._fields(), key=lambda field: ''.join([str(isinstance(field, Measure)), field.name]))
     # end generators
 
-    def addDimension(self,d: (str,Dimension),type: str = 'string'): 
+    def addDimension(self,d: Tuple[str,Dimension],type: str = 'string'): 
         """
         Add a dimension object, or add a string DB column you would like added 
 
@@ -1126,7 +1133,7 @@ def sum(self,field: Dimension):
         """
         self + f'''
             measure: {field.name}_sum {{
-                type: tum
+                type: sum
                 sql: {field.__ref__} ;;
             }} 
         '''
@@ -1140,7 +1147,9 @@ def sumAllNumDimensions(self):
             View: returns your View for method chaining view.sumAllNumDimensions().addCount()...
         """
         for numDim in self.getFieldsByType('number'):
-            self.sum(numDim)
+            # Avoid summing id fields that were mistakenly cast as numbers
+            if '_id' not in numDim.name and numDim.name != 'id':
+                self.sum(numDim)
         return self
 
     def setViewLabel(self,label: str):

lookml/lib/lang.py

@@ -8,7 +8,7 @@ class ws:
 
     #size of a list type object before it breaks onto multiple lines. Int for number of items, not string length
     list_multiline_threshold = 4
-    dense_children_threshold = 2
+    dense_children_threshold = 1
     dense_str_len = 25
 
     #Regex Patterns

lookml/lib/language_data/_allowed_children.py

@@ -44,7 +44,8 @@
                 "sql_always_having",
                 "tags",
                 "view_name",
-                "query"
+                "query",
+                "sql_preamble"
         ],
         "dimension": [
                 "case_sensitive",

lookml/lib/language_data/config.py

@@ -5459,5 +5459,16 @@
                                 "strict"
                         ]
                 }
+        },
+        "sql_preamble": {
+                "explore": {
+                        "type": "sql",
+                        "subtype": "sql-block ",
+                        "indent": 2,
+                        "default_value": "",
+                        "docs_url": "Not documented",
+                        "has_allowed_values": False,
+                        "allowed_values": ""
+                }
         }
 }

lookml/lib/project.py

@@ -550,15 +550,15 @@ def delete_file(self, path:str):
         Delete a file by providing the path (vs delete() which takes a file obj)
 
         Args:
-            fl (str): the path within the project of the file to be deleted
+            f (str): the path within the project of the file to be deleted
 
 .. code-block:: python
 
         myFile = proj.new_file('foo.view.lkml')
         proj.delete_file(path='foo.view.lkml')
 
         """
-        self[fl].delete()
+        self[f].delete()
 
 #P1: build the ability to delete folders from filesystem & github
 class ProjectGithub(Project):
@@ -805,7 +805,14 @@ def __init__(
             )
             self._git.clone(self._git_url)
         self._build_index()
+
     # shell
+    def add(self):
+        '''
+        git add to stage all changes in SSH mode
+        '''
+        self._git.add()
+
     def commit(self):
         """
         git commit and git push changes to remote in SSH mode

lookml/lkml/keys.py

@@ -71,6 +71,7 @@
     "sql_step",
     "sql_on",
     "sql",
+    "sql_preamble"
 )
 
 # These are keys that the serializer should quote the value of (e.g. `label: "Label"`).

lookml/tests/files/basic_parsing/wow.view.lkml

@@ -0,0 +1,35 @@
+explore: custom_audience_revenue {
+  sql_preamble:
+  CREATE TEMP FUNCTION
+    get_negative_segments(json STRING)
+    RETURNS ARRAY<STRING>
+    LANGUAGE js AS """
+      if (json ===null){
+          return []
+      }
+      return (
+          JSON.parse(json)
+              .filter(obj => obj.operation === 'EXCLUDE')
+              .map(obj => obj.segment_id)
+              .reduce((prev,curr) => prev.concat(curr),[])
+              .filter(x => !!x)
+      )
+  """;
+
+  CREATE TEMP FUNCTION
+    get_positive_segments(json STRING)
+    RETURNS ARRAY<STRING>
+    LANGUAGE js AS """
+      if (json ===null){
+          return []
+      }
+      return (
+          JSON.parse(json)
+              .filter(obj => obj.operation !== 'EXCLUDE')
+              .map(obj => obj.segment_id)
+              .reduce((prev,curr) => prev.concat(curr),[])
+              .filter(x => !!x)
+      )
+  """;
+    ;;
+}

lookml/tests/files/kitchenSink/kitchenSink.model.lkml

@@ -11,12 +11,24 @@ datagroup: mydatagroup2 {
   label: "Daily"
   description: "Should fire daily just afte midnight pacific" 
 }
-named_value_format: euro_in_thousands {  value_format: "\"€\"0.000,\" K\"" strict_value_format: yes }
-named_value_format: euro_in_millions {  value_format: "\"€\"0.0000,\" M\"" strict_value_format: yes }
+named_value_format: euro_in_thousands {
+  value_format: "\"€\"0.000,\" K\""
+  strict_value_format: yes 
+}
+named_value_format: euro_in_millions {
+  value_format: "\"€\"0.0000,\" M\""
+  strict_value_format: yes 
+}
 map_layer: neighborhoods {  file: "my_neighborhoods.json" }
 map_layer: cities {  file: "cities.json" }
-access_grant: abc {  allowed_values: ["a","b","c"] user_attribute: abc }
-access_grant: xyz {  allowed_values: ["x","y","z"] user_attribute: abc } 
+access_grant: abc {
+  allowed_values: ["a","b","c"]
+  user_attribute: abc 
+}
+access_grant: xyz {
+  allowed_values: ["x","y","z"]
+  user_attribute: abc 
+} 
 
 view: order_items {
   sql_table_name: ecomm.order_items ;; 
@@ -120,7 +132,9 @@ view: order_items {
     hidden: yes
     sql: ${TABLE}.user_id ;; 
   }
-  dimension: reporting_period {  group_label: "Order Date"  sql: CASE
+  dimension: reporting_period {
+    group_label: "Order Date"
+    sql: CASE
         WHEN date_part('year',${created_raw}) = date_part('year',current_date)
         AND ${created_raw} < CURRENT_DATE
         THEN 'This Year to Date'
@@ -129,20 +143,30 @@ view: order_items {
         AND date_part('dayofyear',${created_raw}) <= date_part('dayofyear',current_date)
         THEN 'Last Year to Date'
 
-      END ;; }
-  dimension: days_since_sold { hidden: yes  sql: datediff('day',${created_raw},CURRENT_DATE) ;; }
+      END ;; 
+  }
+  dimension: days_since_sold {
+    hidden: yes
+    sql: datediff('day',${created_raw},CURRENT_DATE) ;; 
+  }
   dimension: months_since_signup {
     view_label: "Orders"
     type: number
     sql: DATEDIFF('month',${users.created_raw},${created_raw}) ;; 
   }
   dimension: status {  sql: ${TABLE}.status ;; }
-  dimension: days_to_process { type: number  sql: CASE
+  dimension: days_to_process {
+    type: number
+    sql: CASE
         WHEN ${status} = 'Processing' THEN DATEDIFF('day',${created_raw},current_date)*1.0
         WHEN ${status} IN ('Shipped', 'Complete', 'Returned') THEN DATEDIFF('day',${created_raw},${shipped_raw})*1.0
         WHEN ${status} = 'Cancelled' THEN NULL
-      END ;; }
-  dimension: shipping_time { type: number  sql: datediff('day',${shipped_raw},${delivered_raw})*1.0 ;; }
+      END ;; 
+  }
+  dimension: shipping_time {
+    type: number
+    sql: datediff('day',${shipped_raw},${delivered_raw})*1.0 ;; 
+  }
   dimension: sale_price {
     type: number
     value_format_name: usd
@@ -176,7 +200,10 @@ view: order_items {
       ]
     style: interval 
   }
-  dimension: is_returned { type: yesno  sql: ${returned_raw} IS NOT NULL ;; }
+  dimension: is_returned {
+    type: yesno
+    sql: ${returned_raw} IS NOT NULL ;; 
+  }
   dimension: days_until_next_order {
     type: number
     view_label: "Repeat Purchase Facts"

lookml/tests/files/pylookml_test_project/scratch/subfolder/pylookml_scratch.view.lkml

@@ -1,5 +1,8 @@
 
 
 view: pylookml_scratch {
-  dimension: test {  description: "direct filesystem lookml/tests/files/pylookml_test_project local filesystem"  sql: ${hello} ;; } 
+  dimension: test {
+    description: "direct filesystem lookml/tests/files/pylookml_test_project local filesystem"
+    sql: ${hello} ;; 
+  } 
 }

lookml/tests/main_test.py

@@ -733,6 +733,7 @@ def pylookml_test_add_file(self,proj):
         nf.write()
 
     def pylookml_test_delete_file(self,proj):
+        x = proj.new_file('hello.view.lkml')
         f = proj.file('hello.view.lkml')
         f.delete()
         x = proj.new_file('hello2.view.lkml')
@@ -764,6 +765,10 @@ def pylookml_test_mutate_file(self,proj):
     def pylookml_test_project_routine(self,proj):
         # self.assertEqual(len(list(proj.view_files())),19)
         #access a file's deep object via [] syntax
+        # try:
+        self.pylookml_test_delete_file(proj)
+        # except:
+        #     pass
         val_a = proj['views/01_order_items.view.lkml']['views']['order_items']['order_id'].action[0].url.value
         self.assertEqual(val_a,'https://hooks.zapier.com/hooks/catch/1662138/tvc3zj/')
         #access a file's deep object via .file() syntax
@@ -1119,6 +1124,15 @@ def test_add_orphan_to_github(self):
         proj.put(f)
         f.delete()
 
+    def test_cool(self):
+        # x = lkml.load('lookml/tests/files/basic_parsing/wow.view.lkml')
+        # print(x)
+        # with open('lookml/tests/files/basic_parsing/wow.view.lkml','r') as z:
+        #     wow = lkml.load(z)
+        #     print(wow)
+        x = lookml.File('lookml/tests/files/basic_parsing/wow.view.lkml')
+        print(x)
+
     # def test_join_back_an_ndt(self):
     #     v = lookml.View('order_items')
     #     v + f'''

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lookml"
-version = "3.0.3"
+version = "3.0.8"
 description = "A batteries included API for automating your LookML"
 authors = ["Russell Garner"]
 license = "MIT License"

utils/lang_map_generator/lang_map_generator.py

@@ -377,6 +377,7 @@
 p.update({"default_locale":{"localization_settings":{"type":"options","subtype":"options","indent":1,"default_value":"","docs_url":"https://docs.looker.com/reference/manifest-params/localization_settings?version=7.14&lookml=new","has_allowed_values":True,"allowed_values":['ar_AE','ar_JO','ar_SY','hr_HR','fr_BE','es_PA','mt_MT','es_VE','bg','zh_TW','it','ko','uk','lv','da_DK','es_PR','vi_VN','en_US','sr_ME','sv_SE','es_BO','en_SG','ar_BH','pt','ar_SA','sk','ar_YE','hi_IN','ga','en_MT','fi_FI','et','sv','cs','sr_BA_#Latn','el','uk_UA','hu','fr_CH','in','es_AR','ar_EG','ja_JP_JP_#u-ca-japanese','es_SV','pt_BR','be','is_IS','cs_CZ','es','pl_PL','tr','ca_ES','sr_CS','ms_MY','hr','lt','es_ES','es_CO','bg_BG','sq','fr','ja','sr_BA','is','es_PY','de','es_EC','es_US','ar_SD','en','ro_RO','en_PH','ca','ar_TN','sr_ME_#Latn','es_GT','sl','ko_KR','el_CY','es_MX','ru_RU','es_HN','zh_HK','no_NO_NY','hu_HU','th_TH','ar_IQ','es_CL','fi','ar_MA','ga_IE','mk','tr_TR','et_EE','ar_QA','sr__#Latn','pt_PT','fr_LU','ar_OM','th','sq_AL','es_DO','es_CU','ar','ru','en_NZ','sr_RS','de_CH','es_UY','ms','el_GR','iw_IL','en_ZA','th_TH_TH_#u-nu-thai','hi','fr_FR','de_AT','nl','no_NO','en_AU','vi','nl_NL','fr_CA','lv_LV','de_LU','es_CR','ar_KW','sr','ar_LY','mt','it_CH','da','de_DE','ar_DZ','sk_SK','lt_LT','it_IT','en_IE','zh_SG','ro','en_CA','nl_BE','no','pl','zh_CN','ja_JP','de_GR','sr_RS_#Latn','iw','en_IN','ar_LB','es_NI','zh','mk_MK','be_BY','sl_SI','es_PE','in_ID','en_GB']}}})
 p.update({"localization_level":{"localization_settings":{"type":"options","subtype":"options","indent":1,"default_value":"permissive","docs_url":"https://docs.looker.com/reference/manifest-params/localization_settings?version=7.14&lookml=new","has_allowed_values":True,"allowed_values":["permissive","strict"]}}})
 p["query"].update({"explore":{"type":"named_construct","subtype":"identifier","indent":2,"default_value":"","docs_url":"Not documented as child of explore, see https://docs.looker.com/reference/explore-params/aggregate_table?version=7.14&lookml=new#query ","has_allowed_values":False,"allowed_values":""}})
+p.update({"sql_preamble":{"explore":{"type":"sql","subtype":"sql-block ","indent":2,"default_value":"","docs_url":"Not documented","has_allowed_values":False,"allowed_values":""}}})
 #End Generated Code
 
 # generate cfg