-
Notifications
You must be signed in to change notification settings - Fork 0
/
sentiment_analysis.py
72 lines (57 loc) · 2.02 KB
/
sentiment_analysis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import joblib
from transformers import pipeline
from snowflake.snowpark import Session
from snowflake.snowpark.functions import pandas_udf
from snowflake.snowpark.types import StringType, PandasSeriesType
import cachetools
import sys
import pandas as pd
from transformers import pipeline
sentiment_analysis_model = pipeline("sentiment-analysis", model='distilbert-base-uncased-finetuned-sst-2-english')
results=sentiment_analysis_model("I love the way we implement ML models at Infostrux")
joblib.dump(sentiment_analysis_model, 'sentiment-analysis.joblib')
session = Session.builder.configs({
"account": "your_account",
"user": "your_user",
"password": "password",
"role": "SYSADMIN",
"warehouse": "SNOWPARK_DEMO",
"database": "ML_DEMO_DB",
"schema": "ML_DEMO_SCHEMA"
}
).create()
session.file.put(
'sentiment-analysis.joblib',
stage_location = f'@ML_DEMO_DB.ML_DEMO_SCHEMA.my_pretrained_models_stage',
overwrite=True,
auto_compress=False
)
@cachetools.cached(cache={})
def read_model():
import_dir = sys._xoptions.get("snowflake_import_directory")
if import_dir:
# Load the model
return joblib.load(f'{import_dir}/sentiment-analysis.joblib')
@pandas_udf(
name='ML_DEMO_DB.ML_DEMO_SCHEMA.get_sentiment',
session=session,
is_permanent=True,
replace=True,
imports=[
f'@ML_DEMO_DB.ML_DEMO_SCHEMA.my_pretrained_models_stage/sentiment-analysis.joblib'
],
input_types=[PandasSeriesType(StringType())],
return_type=PandasSeriesType(StringType()),
stage_location='@ML_DEMO_DB.ML_DEMO_SCHEMA.my_pretrained_models_stage',
packages=['cachetools==4.2.2', 'transformers==4.14.1']
)
def get_setiment(sentences):
# Load the sentiment analysis model from stage
# using the caching mechanism
sentiment_analysis_model = read_model()
# Apply the model
predictions = []
for sentence in sentences:
result = sentiment_analysis_model(sentence)
predictions.append(result)
return pd.Series(predictions)