Merge pull request #569 from spencermountain/dev

Dev

changelog.md

@@ -4,6 +4,12 @@
 
 -->
 
+#### 10.2.1 [Nov 2023]
+
+- **[change]** - support more templates
+- **[change]** - support multiple citations inside a ref tag
+- **[new]** - add `revisionID()` - thanks Dag-Inge! #568
+
 #### 10.2.0 [Oct 2023]
 
 - **[change]** - typescript export helpers

package.json

@@ -1,7 +1,7 @@
 {
   "name": "wtf_wikipedia",
   "description": "parse wikiscript into json",
-  "version": "10.2.0",
+  "version": "10.2.1",
   "main": "src/index.js",
   "module": "builds/wtf_wikipedia.mjs",
   "unpkg": "builds/wtf_wikipedia-client.min.js",

scratch.js

@@ -1,12 +1,18 @@
 import wtf from './src/index.js'
-console.log('start')
 
-let str = `before {{lit|a|b}} after
+let str = `
+<ref name="cool">{{cite book |last= Bushnell|first= Ian}}
+ {{cite book |last= Walker|first= James W. St. G.}}</ref>
 
 `
 
+str = `<ref> Chapman {{Foo}} </ref> `
+// str = `{{Refplease|date=November 2023|reason=Your explanation here}} in [[Jolgeh-ye Musaabad Rural District]],`
+
 let doc = wtf(str)
-// const doc = await wtf.fetch('https://en.wikivoyage.org/wiki/Interstate_5')
+// const doc = await wtf.fetch('Grand Bend')
 
 // console.log(doc.template().json())
-console.log(doc.text())
+// console.log(doc.text())
+console.log(doc.references().map((r) => r.json()))
+// console.log(doc.templates().map((r) => r.json()))

src/01-document/Document.js

@@ -48,6 +48,7 @@ class Document {
       // userAgent is used for successive calls to the API
       userAgent: options.userAgent || options['User-Agent'] || options['Api-User-Agent'] || 'User of the wtf_wikipedia library',
       templateFallbackFn: options.templateFallbackFn || null,
+      revisionID: options.revisionID || null,
     }
     // this._missing_templates = {} //for stats+debugging purposes
 
@@ -501,6 +502,20 @@ class Document {
     return this
   }
 
+  /**
+   * If a revisionID is given then it sets the revisionID and returns the given revisionID
+   * Else if the revisionID is already set it returns the revisionID
+   *
+   * @param {number} [id] The revisionID that will be set
+   * @returns {number|null} The given or found revisionID
+   */
+  revisionID(id) {
+    if (id !== undefined) {
+      this._revisionID = id
+    }
+    return this._revisionID || null
+  }
+
   options() {
     return this._options
   }

src/02-section/heading.js

@@ -15,7 +15,6 @@ const doInlineTemplates = function (wiki, doc) {
   return wiki
 }
 
-
 /**
  * @typedef fakeSection
  * @property {string} title

src/_version.js

@@ -1 +1 @@
-export default '10.2.0'
+export default '10.2.1'

src/reference/index.js

@@ -1,6 +1,7 @@
 import parse from '../template/parse/toJSON/index.js'
 import { fromText as parseSentence } from '../04-sentence/index.js'
 import Reference from './Reference.js'
+import getTemplates from '../template/find/02-flat.js'
 
 //structured Cite templates - <ref>{{Cite..</ref>
 const hasCitation = function (str) {
@@ -30,32 +31,48 @@ const parseRefs = function (section) {
   let references = []
   let wiki = section._wiki
 
-  wiki = wiki.replace(/ ?<ref>([\s\S]{0,1800}?)<\/ref> ?/gi, function (all, tmpl) {
-    if (hasCitation(tmpl)) {
-      let obj = parseCitation(tmpl)
-      if (obj) {
-        references.push({ json: obj, wiki: all })
+  wiki = wiki.replace(/ ?<ref>([\s\S]{0,1800}?)<\/ref> ?/gi, function (all, txt) {
+    let found = false
+    // there could be more than 1 template inside a <ref><ref>
+    let arr = getTemplates(txt)
+    arr.forEach((tmpl) => {
+      if (hasCitation(tmpl)) {
+        let obj = parseCitation(tmpl)
+        if (obj) {
+          references.push({ json: obj, wiki: all })
+          found = true
+        }
+        wiki = wiki.replace(tmpl, '')
       }
-      wiki = wiki.replace(tmpl, '')
-    } else {
-      references.push({ json: parseInline(tmpl), wiki: all })
+    })
+    // parse as an inline reference
+    if (!found) {
+      references.push({ json: parseInline(txt), wiki: all })
     }
     return ' '
   })
 
   //<ref name=""/>
   wiki = wiki.replace(/ ?<ref [^>]{0,200}?\/> ?/gi, ' ')
 
-  //<ref name=""></ref>
-  wiki = wiki.replace(/ ?<ref [^>]{0,200}>([\s\S]{0,1800}?)<\/ref> ?/gi, function (all, tmpl) {
-    if (hasCitation(tmpl)) {
-      let obj = parseCitation(tmpl)
-      if (obj) {
-        references.push({ json: obj, wiki: tmpl })
+  //<ref name=""></ref> (a gross copy of above parsing)
+  wiki = wiki.replace(/ ?<ref [^>]{0,200}>([\s\S]{0,1800}?)<\/ref> ?/gi, function (all, txt) {
+    let found = false
+    // there could be more than 1 template inside a <ref><ref>
+    let arr = getTemplates(txt)
+    arr.forEach((tmpl) => {
+      if (hasCitation(tmpl)) {
+        let obj = parseCitation(tmpl)
+        if (obj) {
+          references.push({ json: obj, wiki: all })
+          found = true
+        }
+        wiki = wiki.replace(tmpl, '')
       }
-      wiki = wiki.replace(tmpl, '')
-    } else {
-      references.push({ json: parseInline(tmpl), wiki: all })
+    })
+    // parse as an inline reference
+    if (!found) {
+      references.push({ json: parseInline(txt), wiki: all })
     }
     return ' '
   })

src/template/custom/aliases.js

@@ -261,6 +261,69 @@ let multi = {
   'video game release': ['vgrelease', 'video game release hlist', 'vgrtbl', 'vgrelease hlist', 'vgrh'],
   aka: ['a.k.a.', 'also known as'],
   'literal translation': ['lit', 'literal', 'literally'],
+  'citation needed': [
+    'are you sure?',
+    'cb',
+    'ciation needed',
+    'cit',
+    'cita requerida',
+    'citaiton needed',
+    'citation missing',
+    'citation need',
+    'citation requested',
+    'citation required',
+    'citation-needed',
+    'citationeeded',
+    'citationneeded',
+    'citationrequired',
+    'citazione necessaria',
+    'cite missing',
+    'cite needed',
+    'cite source',
+    'cite-needed',
+    'citeneeded',
+    'citesource',
+    'citn',
+    'cn needed',
+    'cn',
+    'ctn',
+    'fact?',
+    'fact',
+    'facts',
+    'fcitation needed',
+    'me-fact',
+    'need citation',
+    'need sources',
+    'need-ref',
+    'needcitation',
+    'needcite',
+    'needs citation',
+    'needs citations',
+    'needs reference',
+    'needs source',
+    'needs-cite',
+    'needsref',
+    'no source given',
+    'prov-statement',
+    'prove it',
+    'proveit',
+    'ref needed',
+    'ref-needed',
+    'ref?',
+    'reference necessary',
+    'reference needed',
+    'reference required',
+    'refnec',
+    'refneeded',
+    'refplease',
+    'request citation',
+    'source needed',
+    'source?',
+    'sourceme',
+    'uncited',
+    'unreferenced inline',
+    'unsourced-inline',
+  ],
 }
 
 // - other languages -

src/template/custom/data-only/dummy-templates.js

@@ -0,0 +1,14 @@
+import parse from '../../parse/toJSON/index.js'
+
+// dummy templates that get parsed properly already,
+// but present here for for aliases + template coverage tests
+let templates = {}
+let dummies = ['citation needed']
+dummies.forEach((name) => {
+  // just parse it and do nothing
+  templates[name] = (tmpl, list) => {
+    list.push(parse(tmpl))
+    return ''
+  }
+})
+export default templates

src/template/custom/data-only/functions.js

@@ -232,7 +232,7 @@ export default {
       let positionalIndex = 0
       for (let i = 0; i < parameters.length; i++) {
         let parameter = parameters[i].trim()
-        if (parameter.match(/^[a-zA-Z_]/)) {
+        if (parameter.match(/^[a-z_]/i)) {
           // Named argument
           let [key, value] = parameter.split('=')
           // At this point, the Lua code evaluates alttot1 and alttot2 values as
@@ -398,7 +398,7 @@ export default {
     return ''
   },
   //https://en.wikipedia.org/wiki/Template:MedalCount
-  'medalcount': (tmpl, list) => {
+  medalcount: (tmpl, list) => {
     let all = parse(tmpl).list || []
     let lines = []
     for (let i = 0; i < all.length; i += 4) {
@@ -411,9 +411,9 @@ export default {
     }
     let obj = {
       template: 'medalcount',
-      list: lines
+      list: lines,
     }
     list.push(obj)
     return ''
-  }
+  },
 }

src/template/custom/data-only/index.js

@@ -1,3 +1,8 @@
+import idName from './id-name.js'
+import idTitle from './id-title.js'
+import dummies from './dummy-templates.js'
+import fns from './functions.js'
+
 let templates = {
   //https://en.wikipedia.org/wiki/Category:External_link_templates
   'find a grave': ['id', 'name', 'work', 'last', 'first', 'date', 'accessdate'],
@@ -12,19 +17,9 @@ let templates = {
   'taxon info': ['taxon', 'item'], //https://en.wikipedia.org/wiki/Template:Taxon_info
   'portuguese name': ['first', 'second', 'suffix'], // https://en.wikipedia.org/wiki/Template:Portuguese_name
   geo: ['lat', 'lon', 'zoom'], //https://en.wikivoyage.org/wiki/Template:Geo
-  hatnote: ['text']
+  hatnote: ['text'],
 }
 
-
-import idName from './id-name.js'
-import idTitle from './id-title.js'
-import fns from './functions.js'
-
-templates = Object.assign(
-  templates,
-  idName,
-  idTitle,
-  fns
-)
+templates = Object.assign(templates, dummies, idName, idTitle, fns)
 
 export default templates

src/template/custom/index.js

@@ -3,17 +3,12 @@ import textTmpl from './text-only/index.js'
 import dataTmpl from './data-only/index.js'
 import bothTmpl from './text-and-data/index.js'
 
-let templates = Object.assign(
-  {},
-  textTmpl,
-  dataTmpl,
-  bothTmpl
-)
+let templates = Object.assign({}, textTmpl, dataTmpl, bothTmpl)
 
 Object.keys(aliases).forEach((k) => {
-  // if (templates[aliases[k]] === undefined) {
-  //   console.error(`Missing template: '${aliases[k]}'`)
-  // }
+  if (templates[aliases[k]] === undefined) {
+    console.error(`Missing template: '${aliases[k]}'`)
+  }
   templates[k] = templates[aliases[k]]
 })
 export default templates

src/template/custom/text-and-data/functions.js

@@ -355,10 +355,26 @@ let templates = {
       txt = obj.list[0] || ''
     }
     // replace double quotes with singles and put the text inside double quotes
-    let result = txt.replace(/"/g, '\'')
+    let result = txt.replace(/"/g, "'")
     result = '"' + result + '"'
     return result
   },
 
+  // https://de.m.wikipedia.org/wiki/Vorlage:ReptileDatabase
+  ReptileDatabase: (tmpl, list) => {
+    let obj = parse(tmpl, ['taxon', 'genus', 'species', 'abruf', 'pure_url'])
+    list.push(obj)
+    let str = ''
+    if (obj.genus || obj.species) {
+      str = `${obj.genus || ''} ${obj.species || ''} `
+    }
+    return `${str}In: [[The Reptile Database]]`
+  },
+  //https://en.m.wikipedia.org/wiki/Template:GEOnet3
+  GEOnet3: (tmpl, list) => {
+    let obj = parse(tmpl, ['ufi', 'name'])
+    list.push(obj)
+    return `GEOnet3 can be found at [[GEOnet Names Server]], at [http://geonames.nga.mil/namesgaz/ this link]`
+  },
 }
 export default templates

tests/integration/Document.test.js

@@ -56,6 +56,26 @@ test('pageID - get / set - if the pageID is set then it should return the same '
   t.end()
 })
 
+//revisionID
+test('revisionID - get - should initially be null', (t) => {
+  let doc = wtf('')
+  t.equal(doc.revisionID(), null, 'the revisionID equals null')
+  t.end()
+})
+
+test('revisionID - get - if the revisionID is already set than get it from internal object', (t) => {
+  let doc = wtf('', { revisionID: 1 })
+  t.equal(doc.revisionID(), 1, 'the revisionID equals 1')
+  t.end()
+})
+
+test('revisionID - get / set - if the revisionID is set then it should return the same ', (t) => {
+  let doc = wtf('')
+  doc.revisionID(1)
+  t.equal(doc.revisionID(), 1, 'the revisionID equals 1')
+  t.end()
+})
+
 //wikidata
 test('wikidata - get - should initially be null', (t) => {
   let doc = wtf('')