Merge branch 'master' into remove_sdcommon

docs_requirements.txt

@@ -7,7 +7,7 @@ backcall==0.2.0
 bleach==3.2.1
 certifi==2020.12.5
 cffi==1.14.4
-chardet==3.0.4
+chardet==4.0.0
 colorama==0.4.4
 decorator==4.4.2
 defusedxml==0.6.0

qcodes/dataset/guid_helpers.py

@@ -0,0 +1,92 @@
+from typing import Tuple, Union, Dict, Optional, List, Iterable
+
+from pathlib import Path
+import gc
+import re
+
+from sqlite3 import DatabaseError
+
+from qcodes.dataset.sqlite.queries import get_guids_from_run_spec
+from qcodes.dataset.sqlite.database import connect
+
+
+def guids_from_dbs(
+    db_paths: Iterable[Path],
+) -> Tuple[Dict[Path, List[str]], Dict[str, Path]]:
+    """
+    Extract all guids from the supplied database paths.
+
+    Args:
+        db_paths: Path or str or directory where to search
+
+    Returns:
+        Tuple of Dictionary mapping paths to lists of guids as strings
+        and Dictionary mapping guids to db paths.
+    """
+    dbdict = {}
+    for p in db_paths:
+        try:
+            conn = connect(str(p))
+            dbdict[p] = get_guids_from_run_spec(conn)
+        except (RuntimeError, DatabaseError) as e:
+            print(e)
+        finally:
+            conn.close()
+            gc.collect()
+    guiddict = {}
+    for dbpath, guids in dbdict.items():
+        guiddict.update({guid: dbpath for guid in guids})
+    return dbdict, guiddict
+
+
+def guids_from_dir(
+    basepath: Union[Path, str]
+) -> Tuple[Dict[Path, List[str]], Dict[str, Path]]:
+    """
+    Recursively find all db files under basepath and extract guids.
+
+    Args:
+        basepath: Path or str or directory where to search
+
+    Returns:
+        Tuple of Dictionary mapping paths to lists of guids as strings
+        and Dictionary mapping guids to db paths.
+    """
+    return guids_from_dbs(Path(basepath).glob("**/*.db"))
+
+
+def guids_from_list_str(s: str) -> Optional[Tuple[str, ...]]:
+    """
+    Get tuple of guids from a python/json string representation of a list.
+
+    Extracts the guids from a string representation of a list, tuple,
+    or set of guids or a single guid.
+
+    Args:
+        s: input string
+
+    Returns:
+        Extracted guids as a tuple of strings.
+        If a provided string does not match the format, `None` will be returned.
+        For an empty list/tuple/set or empty string an empty set is returned.
+    Examples:
+        >>> guids_from_str(
+        "['07fd7195-c51e-44d6-a085-fa8274cf00d6', \
+          '070d7195-c51e-44d6-a085-fa8274cf00d6']")
+        will return
+        ('07fd7195-c51e-44d6-a085-fa8274cf00d6',
+        '070d7195-c51e-44d6-a085-fa8274cf00d6')
+    """
+    guid = r"[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}"
+    captured_guid = fr"(?:\s*['\"]?({guid})['\"]?\s*)"
+    open_parens = r"[\[\(\{]"
+    close_parens = r"[\]\)\}]"
+    m = re.match(
+        fr"^\s*{open_parens}?"
+        fr"(?:{captured_guid},)*{captured_guid}?"
+        fr"{close_parens}?\s*$",
+        s,
+    )
+    if m is None:
+        return None
+    return tuple(v for v in m.groups() if v is not None)

qcodes/dataset/guids.py

@@ -1,11 +1,13 @@
 from typing import Union, Dict, Sequence, List, Optional
+
 import time
 import re
 
-import numpy as np
 
+import numpy as np
 import qcodes as qc
 
+
 _guid_pattern = re.compile(r'^[\da-f]{8}-([\da-f]{4}-){3}[\da-f]{12}$')
 
 

qcodes/tests/dataset/test_guid_helpers.py

@@ -0,0 +1,59 @@
+from typing import cast
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+
+from qcodes.dataset.experiment_container import new_experiment
+from qcodes.dataset.measurements import Measurement
+from qcodes.dataset.sqlite.database import initialised_database_at
+from qcodes.instrument.parameter import Parameter
+
+
+from qcodes.dataset.guid_helpers import guids_from_dir, guids_from_list_str
+
+
+def test_guids_from_dir(tmp_path: Path) -> None:
+    def generate_local_run(dbpath: Path) -> str:
+        with initialised_database_at(str(dbpath)):
+            new_experiment(sample_name="fivehundredtest_sample",
+                           name="fivehundredtest_name")
+
+            p1 = Parameter('Voltage', set_cmd=None)
+            p2 = Parameter('Current', get_cmd=np.random.randn)
+
+            meas = Measurement()
+            meas.register_parameter(p1).register_parameter(p2, setpoints=[p1])
+
+            with meas.run() as datasaver:
+                for v in np.linspace(0, 2, 250):
+                    p1(v)
+                    datasaver.add_result((p1, cast(float, p1())),
+                                         (p2, cast(float, p2())))
+            guid = datasaver.dataset.guid
+            datasaver.flush_data_to_database(block=True)
+        return guid
+
+    paths_counts = [
+        (tmp_path / 'subdir1' / 'dbfile1.db', 2),
+        (tmp_path / 'subdir1' / 'dbfile2.db', 4),
+        (tmp_path / 'subdir2' / 'dbfile1.db', 1),
+        (tmp_path / 'dbfile1.db', 3),
+    ]
+    guids = defaultdict(list)
+    for path, count in paths_counts:
+        path.parent.mkdir(exist_ok=True, parents=True)
+        for _ in range(count):
+            guids[path].append(generate_local_run(path))
+    dbdict, _ = guids_from_dir(tmp_path)
+    assert dbdict == guids
+
+
+def test_guids_from_list_str() -> None:
+    guids = ['07fd7195-c51e-44d6-a085-fa8274cf00d6',
+             '070d7195-c51e-44d6-a085-fa8274cf00d6']
+    assert guids_from_list_str('') == tuple()
+    assert guids_from_list_str(str(guids)) == tuple(guids)
+    assert guids_from_list_str(str([guids[0]])) == (guids[0],)
+    assert guids_from_list_str(str(tuple(guids))) == tuple(guids)
+    assert guids_from_list_str(str(guids[0])) == (guids[0],)

qcodes/tests/dataset/test_guids.py

@@ -1,20 +1,22 @@
 import time
+
 from uuid import uuid4
 
-import pytest
-from hypothesis import given, settings, assume
 import hypothesis.strategies as hst
+from hypothesis import assume, given, settings
+
 import numpy as np
+import pytest
 
 import qcodes as qc
-from qcodes.dataset.guids import (generate_guid, parse_guid,
-                                  set_guid_location_code,
+from qcodes.dataset.guids import (filter_guids_by_parts, generate_guid,
+                                  parse_guid, set_guid_location_code,
                                   set_guid_work_station_code,
-                                  validate_guid_format,
-                                  filter_guids_by_parts)
+                                  validate_guid_format)
 from qcodes.tests.common import default_config
 
 
+
 @settings(max_examples=50, deadline=1000)
 @given(loc=hst.integers(0, 255), stat=hst.integers(0, 65535),
        smpl=hst.integers(0, 4294967295))

requirements.txt

@@ -7,7 +7,7 @@ broadbean~=0.9.1
 cachetools~=4.2.0
 certifi~=2020.12.5
 cffi~=1.14.4
-chardet~=3.0.4
+chardet~=4.0.0
 colorama~=0.4.4
 contextlib2~=0.6.0.post1
 cycler~=0.10.0

science_requirements.txt

@@ -11,7 +11,7 @@ bcrypt==3.2.0
 black==20.8b1
 bleach==3.2.1
 cffi==1.14.4
-chardet==3.0.4
+chardet==4.0.0
 click==7.1.2
 cloudpickle==1.6.0
 colorama==0.4.4

test_requirements.txt

@@ -2,7 +2,7 @@ apipkg==1.5
 atomicwrites==1.4.0
 attrs==20.3.0
 certifi==2020.12.5
-chardet==3.0.4
+chardet==4.0.0
 codacy-coverage==1.3.11
 colorama==0.4.4
 coverage==5.3