Initial commit

.ci/pythonStdlibFixes.sh

@@ -0,0 +1,2 @@
+if $( python -c "import sys;sys.exit(int(not (sys.version_info < (3, 5)) ))" ); then curl -O https://raw.githubusercontent.com/python/cpython/3.6/Lib/typing.py; fi;
+if $( python -c "import sys;sys.exit(int(not (sys.version_info < (3, 6)) ))" ); then curl -O https://raw.githubusercontent.com/python/cpython/3.7/Lib/enum.py; fi;

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -0,0 +1 @@
+*.model filter=lfs diff=lfs merge=lfs -text

.github/.templateMarker

@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"

.github/workflows/CI.yml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,9 @@
+__pycache__
+*.pyc
+*.pyo
+/HDDModelDecoder.egg-info
+/build
+/dist
+/.eggs
+/HDDModelDecoder/ml/WD_models/*.json
+/HDDModelDecoder/ml/WD_models/*.model

.gitlab-ci.yml

@@ -0,0 +1,76 @@
+#image: pypy:latest
+image: registry.gitlab.com/kolanich/fixed_python:latest
+stages:
+  - dependencies
+  - build
+  - test
+  - tooling
+
+build:
+  tags:
+    - shared
+  stage: build
+  variables:
+    GIT_DEPTH: "1"
+    PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+
+  before_script:
+    - export PYTHON_MODULES_DIR=${PYTHONUSERBASE}/lib/python3.7
+    - export EXECUTABLE_DEPENDENCIES_DIR=${PYTHONUSERBASE}/bin
+    - export PATH="$PATH:$EXECUTABLE_DEPENDENCIES_DIR" # don't move into `variables` any of them, it is unordered
+    - pip3 install --user --pre --upgrade git+https://github.com/berkerpeksag/astor.git git+https://github.com/erikrose/more-itertools.git
+    - pip3 install --user --pre --upgrade https://gitlab.com/KOLANICH/pyxgboost/-/jobs/artifacts/master/raw/wheels/pyxgboost-0.CI-py3-none-any.whl?job=build
+    - pip3 install --user --pre --upgrade git+https://gitlab.com/KOLANICH/AutoXGBoost.py.git git+https://gitlab.com/KOLANICH/NoSuspend.py.git
+
+  script:
+    - mkdir wheels
+    - python3 learn/learnWDSeries.py --train --score 10
+    - python3 setup.py bdist_wheel
+    - coverage run --source=HDDModelDecoder tests/test.py || true
+    - ls -l ./dist
+    - mv ./dist/*.whl ./wheels/HDDModelDecoder-0.CI_xgboost-py3-none-any.whl
+    - pip3 install --upgrade --pre --user ./wheels/HDDModelDecoder-0.CI_xgboost-py3-none-any.whl
+
+    - rm HDDModelDecoder/ml/WD_models/*.model
+    - python3 learn/learnWDSeries.py --train --score 10 --format python
+    - python3 setup.py bdist_wheel
+    - coverage run --source=HDDModelDecoder -m pytest --junitxml=./rspec.xml ./tests/tests.py || true
+    - coverage report -m
+    - coverage xml
+    - ls -l ./dist
+    - mv ./dist/*.whl ./wheels/HDDModelDecoder-0.CI_python-py3-none-any.whl
+    - pip3 install --upgrade --pre --user ./wheels/HDDModelDecoder-0.CI_python-py3-none-any.whl
+
+  coverage: /^TOTAL\\s+.+?(\\d{1,3}%)$/
+
+  cache:
+    paths:
+      - $PYTHONUSERBASE
+
+  artifacts:
+    paths:
+      - dist
+    reports:
+      junit: ./rspec.xml
+      cobertura: ./coverage.xml
+
+checks:
+  stage: tooling
+  tags:
+    - shared
+  image: docker:latest
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  services:
+    - docker:dind
+  script:
+    - docker run --env SAST_CONFIDENCE_LEVEL=5 --volume "$PWD:/code" --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/sast:latest" /app/bin/run /code
+    #- docker run --env SOURCE_CODE="$PWD" --env CODECLIMATE_VERSION="latest" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/codequality:latest" /code
+    #- docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" --volume "$PWD:/code" --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:latest" /code
+
+  artifacts:
+    reports:
+      #codequality: gl-code-quality-report.json
+      sast: gl-sast-report.json
+      #dependency_scanning: gl-dependency-scanning-report.json 

Code_Of_Conduct.md

@@ -0,0 +1 @@
+No codes of conduct!

HDDModelDecoder/HGST.py

@@ -0,0 +1,144 @@
+__all__=("HGSTDecoder",)
+import re
+from .interfaces import *
+
+vendors={
+	"H":"HGST",
+	"W":"WD"
+}
+families={
+	"U": "Ultrastar",
+	"D": "Deskstar",
+	"T": "Travelstar",
+	"E": "Endurastar",
+	"C": "CinemaStar",
+	"M": "Microdrive",
+	"S": "Shingled Magnetic Recording"
+}
+
+series = {
+	"H": "Helium",
+	"S": "Standard",
+	"C": "Compact",
+	"E": "Enhanced Availability",
+	"A": "Auto",
+	"T": "Two-disk",
+	"N": "NAS",
+	"P": "P Series"
+}
+heights={
+	"L": 1,
+	"7": 0.276,
+	"9": 0.374,
+	"5": 0.197,
+	"S": 0.583
+}
+
+interfaces={
+	"16": IF(SCSI, speed=0.160, pin_count=68),
+	"18": IF(SCSI, speed=0.160, pin_count=80),
+	"36": IF(SCSI, speed=0.320, pin_count=68),
+	"38": IF(SCSI, speed=0.320, pin_count=80),
+	"F2": IF(FC, speed=2, pin_count=40),
+	"F4": IF({}, interface="FCAL", speed=4),
+	"AT": PATA,
+	"SA": SATA150,
+	"A3": SATA300,
+	"A6": SATA600,
+	"E6": SATA600,#512e,
+	"N6": SATA600,#4Kn
+	"SS": SAS,
+	"42": SAS3,#4Kn,
+	"52": SAS3,#512e,
+	"S6": SAS2,
+}
+
+securityModes={
+	"0":"Instant Secure Erase",
+	"1":"Bulk Data Encryption (SATA), TCG SED encryption (SAS)",
+	"2": None, # reserved
+	"4":"Secure Erase (overwrite only)",
+	"5":"TCG encryption with FIPS (SAS)",
+}
+
+generationCodesToNumberOfPlatters={
+	"P":1,
+	"D":2,
+	"V":3,
+	"K":5,
+	"A":8,
+}
+buffer_sizes={# MiB
+	"0":None,#reserved
+	"1":128,
+	"2":2,
+	"3":32,
+	"4":64,
+	"8":8,
+	"6":16
+}
+featureCodes={
+	"M": "Host-managed",
+	"0": "Reserved",
+	"L": "Legacy Pin 3 configuration",
+}
+
+coolSpin={
+	"5C":{"variable_rpm":True}
+}
+
+
+vendorRx="(["+"".join(vendors.keys())+"])"
+familiesRx="(["+"".join(families.keys())+"])"
+seriesRx="(["+"".join(series.keys())+"])"
+rpmRx="(\d{2}|"+"|".join(coolSpin.keys())+")"
+capacityRx="(\d{2})"
+generationCodesRx="(["+"".join(generationCodesToNumberOfPlatters.keys())+"])"
+heightsRx="(["+"".join(heights.keys())+"])"
+interfacesRx="("+"|".join(interfaces.keys())+")"
+featureCodesRx="(["+"".join(buffer_sizes.keys())+"".join(featureCodes.keys())+"])"
+securityModesRx="(["+"".join(securityModes.keys())+"])"
+modelRx=re.compile(vendorRx+familiesRx+seriesRx+rpmRx+capacityRx+capacityRx+generationCodesRx+heightsRx+interfacesRx+featureCodesRx+securityModesRx)
+
+def HGSTDecoder(model:str):
+	"""Decoder for HGST and Hitachi drives
+	for more details visit the search results of 
+	https://www.google.com/search?q=inurl%3Ahttps%3A%2F%2Fwww.hgst.com%2Fsites%2Fdefault%2Ffiles%2Fresources%2F+filetype%3Apdf+%22How+to+read%22+%22model+number%22
+	"""
+	dic={}
+	dic["vendor"]=vendors[model[0]]
+	dic["family"]=families[model[1]]
+	dic["series"]=series[model[2]]
+
+	if model[3:5] not in coolSpin:
+		dic["rpm"]=int(model[3:5])
+		if dic["rpm"]<36:
+			dic["rpm"]*=1000
+		else:
+			dic["rpm"]*=100
+	else:
+		dic.update(coolSpin[model[3:5]])
+
+	capacityPowerOf10=100
+	if model[1] == "S" :
+	 	capacityPowerOf10*=10
+	dic["top_capacity"]=int(model[5:7]) * capacityPowerOf10 #the capacity, in gigabytes, of the largest member of the drive family, divided by 10 (or 100).
+	dic["capacity"]=int(model[7:9]) * capacityPowerOf10 #Capacity this model, GB
+	dic["generation_code"]=model[9] # Generation code, although this value can also be used to identify the maximum number of platters in the drive family
+	if dic["generation_code"] in generationCodesToNumberOfPlatters:
+		dic["max_platters"]=generationCodesToNumberOfPlatters[dic["generation_code"]]
+
+	dic["form_factor"]={}
+	dic["form_factor"]["height"]=heights[model[10]]
+	dic["interface"]=interfaces[model[11:13]]
+	dic["feature_code"]=model[13]
+	if dic["feature_code"] in buffer_sizes:
+		dic["buffer_size"]=buffer_sizes[dic["feature_code"]]
+	if dic["feature_code"] in featureCodes:
+		dic["feature"]=featureCodes[dic["feature_code"]]
+
+	if model[14] in securityModes:
+		dic["data_security_mode"]=securityModes[model[14]]
+	else:
+		dic["data_security_mode"]=model[14]
+	return dic

HDDModelDecoder/Samsung.py

@@ -0,0 +1,96 @@
+__all__=("SamsungDecoder",)
+import re
+from .interfaces import *
+from collections import OrderedDict
+
+sPostfixes={
+	"N": {"interface":UATA133, "series":"P80"},
+	"S": {"interface":SATA,},
+	"C": {"interface":SATA,},
+}
+
+def SamsungDecoderS(prefix:str, number:str, postfix:str):
+	dic={}
+	dic.update(sPostfixes[postfix[-1]])
+
+	if dic["interface"]["interface"]=="SATA":
+		dic["buffer_size"]=8
+	if postfix[-1]=="N" or postfix[-1]=="C":
+		dic["capacity"]=int(number[0:2])*10
+	else:
+		dic["capacity"]=int(number[1])*10
+	dic["heads"]=int(postfix[0:-1])
+	return dic
+
+segments={
+	"D": "Desktop",
+	"M": "Mobile",
+	"E": "Enterprise"
+}
+
+postfixes=OrderedDict(( # regex working depends on the order
+	("HI", {"interface": SATA300, "rpm": 5400}),
+	("HJ", {"interface": SATA300, "rpm": 7200}),
+	("GJ", {"interface": SATA300, "rpm": 7200}),
+	("HX", {"interface": USB20, "rpm": 5400}),
+	("IX", {"interface": USB20, "rpm": 5400}), # may be wrong
+	("JX", {"interface": USB20, "rpm": 5400}), # may be wrong
+	("JI", {"interface": SATA300, "rpm": 5400}),
+	('HA', {"interface": IF(PATA, ZIF=True), "rpm": 3600}),
+	('GA', {"interface": IF(PATA, ZIF=True), "rpm": 3600}), # may be wrong
+	('GB', {"interface": IF(PATA, ZIF=True), "rpm": 4200}), # may be wrong
+	('GI', {"interface": SATA300, "rpm":  None}), # may be wrong, trouble with rpm
+	('HB', {"interface": IF(PATA, ZIF=True), "rpm": 4200}),
+	('THB', {"interface": IF(PATA, ZIF=True), "rpm": 4200}),
+	('HC', {"interface": PATA, "rpm": 5400}),
+	('II', {"interface": SATA300, "rpm": 5400}),
+	('IJ', {"interface": SATA300, "rpm": 7200}),
+	('JB', {"interface": IF(PATA, ZIF=True), "rpm": 4200}),
+	('TJB', {"interface": IF(PATA, ZIF=True), "rpm": 4200}), # may be wrong
+	('JJ', {"interface": SATA300, "rpm": 7200}),
+	('JQ', {"interface": CEATA, "rpm": 4200}),
+	('UJQ', {"interface": CEATA, "rpm": 4200}), # may be wrong
+	('LD', {"interface": UATA100, "rpm": 7200}),
+	('LI', {"interface": SATA300, "rpm": 5400}),
+	('LJ', {"interface": SATA300, "rpm": 7200}),
+	('MBB', {"interface": SATA300, "rpm": 5400}),
+	('RHF', {"interface": SATA300, "rpm": 5400}),
+	('RJF', {"interface": SATA300, "rpm": 5400}),
+	('SI', {"interface": SATA300}),
+	('SJ', {"interface": SATA300, "rpm": 7200}),
+	('UI', {"interface": SATA300}),
+	('UJ', {"interface": SATA300, "rpm": 7200}),
+	('VHF', {"interface": SATA300, "rpm": 5400}),
+	('VJF', {"interface": SATA300, "rpm": 5400}),
+	('WI', {"interface": SATA300}),
+))
+
+def SamsungDecoderH(prefix:str, number:str, postfix:str):
+	dic={}
+	dic["segment"]=segments[prefix[-1]]
+	dic.update(postfixes[postfix])
+	return dic
+
+def SamsungDecoderHS(prefix:str, number:str, postfix:str):
+	dic={}
+	dic.update(postfixes[postfix])
+	return dic
+
+prefixes={
+	'HD': SamsungDecoderH ,
+	'HE': SamsungDecoderH ,
+	'HM': SamsungDecoderH ,
+	'HN-M': SamsungDecoderH,
+	'HS':SamsungDecoderHS,
+	'SP': SamsungDecoderS
+}
+
+prefixesRx="("+"|".join(prefixes)+")"
+postfixesRx="("+"|".join(postfixes)+"|\\d["+"".join(sPostfixes)+"])"
+modelRx=re.compile(prefixesRx+"(\\d{2,3})"+postfixesRx )
+def SamsungDecoder(model:str):
+	model = model.upper()
+	m = (prefix, num, postfix) = modelRx.match(model).groups()
+	dic = prefixes[prefix](*m)
+	dic["vendor"] = "Samsung"
+	return dic