{"payload":{"allShortcutsEnabled":false,"fileTree":{"docs/tutorials/python_userguide":{"items":[{"name":"attach_model.txt","path":"docs/tutorials/python_userguide/attach_model.txt","contentType":"file"},{"name":"base_plsa.txt","path":"docs/tutorials/python_userguide/base_plsa.txt","contentType":"file"},{"name":"coherence.txt","path":"docs/tutorials/python_userguide/coherence.txt","contentType":"file"},{"name":"different.txt","path":"docs/tutorials/python_userguide/different.txt","contentType":"file"},{"name":"index.txt","path":"docs/tutorials/python_userguide/index.txt","contentType":"file"},{"name":"loading_data.txt","path":"docs/tutorials/python_userguide/loading_data.txt","contentType":"file"},{"name":"m_artm.txt","path":"docs/tutorials/python_userguide/m_artm.txt","contentType":"file"},{"name":"phi_theta_extraction.txt","path":"docs/tutorials/python_userguide/phi_theta_extraction.txt","contentType":"file"},{"name":"ptdw.txt","path":"docs/tutorials/python_userguide/ptdw.txt","contentType":"file"},{"name":"regularizers_and_scores.txt","path":"docs/tutorials/python_userguide/regularizers_and_scores.txt","contentType":"file"}],"totalCount":10},"docs/tutorials":{"items":[{"name":"python_userguide","path":"docs/tutorials/python_userguide","contentType":"directory"},{"name":"bigartm_cli.txt","path":"docs/tutorials/bigartm_cli.txt","contentType":"file"},{"name":"datasets.txt","path":"docs/tutorials/datasets.txt","contentType":"file"},{"name":"index.txt","path":"docs/tutorials/index.txt","contentType":"file"},{"name":"python_tutorial.txt","path":"docs/tutorials/python_tutorial.txt","contentType":"file"},{"name":"regularizers_descr.txt","path":"docs/tutorials/regularizers_descr.txt","contentType":"file"},{"name":"scores_descr.txt","path":"docs/tutorials/scores_descr.txt","contentType":"file"}],"totalCount":7},"docs":{"items":[{"name":"_images","path":"docs/_images","contentType":"directory"},{"name":"_static","path":"docs/_static","contentType":"directory"},{"name":"_templates","path":"docs/_templates","contentType":"directory"},{"name":"api_references","path":"docs/api_references","contentType":"directory"},{"name":"devguide","path":"docs/devguide","contentType":"directory"},{"name":"installation","path":"docs/installation","contentType":"directory"},{"name":"release_notes","path":"docs/release_notes","contentType":"directory"},{"name":"tutorials","path":"docs/tutorials","contentType":"directory"},{"name":"visartm","path":"docs/visartm","contentType":"directory"},{"name":"Makefile","path":"docs/Makefile","contentType":"file"},{"name":"README","path":"docs/README","contentType":"file"},{"name":"conf.py","path":"docs/conf.py","contentType":"file"},{"name":"devguide.txt","path":"docs/devguide.txt","contentType":"file"},{"name":"download.txt","path":"docs/download.txt","contentType":"file"},{"name":"index.txt","path":"docs/index.txt","contentType":"file"},{"name":"intro.txt","path":"docs/intro.txt","contentType":"file"},{"name":"make.bat","path":"docs/make.bat","contentType":"file"},{"name":"requirements.txt","path":"docs/requirements.txt","contentType":"file"}],"totalCount":18},"":{"items":[{"name":"3rdparty","path":"3rdparty","contentType":"directory"},{"name":"cmake_modules","path":"cmake_modules","contentType":"directory"},{"name":"csharp","path":"csharp","contentType":"directory"},{"name":"docs","path":"docs","contentType":"directory"},{"name":"python","path":"python","contentType":"directory"},{"name":"src","path":"src","contentType":"directory"},{"name":"test_data","path":"test_data","contentType":"directory"},{"name":"utils","path":"utils","contentType":"directory"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":".travis.yml","path":".travis.yml","contentType":"file"},{"name":"CMakeLists.txt","path":"CMakeLists.txt","contentType":"file"},{"name":"LICENSE.txt","path":"LICENSE.txt","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"},{"name":"appveyor-mingw.yml","path":"appveyor-mingw.yml","contentType":"file"},{"name":"appveyor.yml","path":"appveyor.yml","contentType":"file"},{"name":"codestyle_checks.sh","path":"codestyle_checks.sh","contentType":"file"}],"totalCount":16}},"fileTreeProcessingTime":13.835588,"foldersToFetch":[],"repo":{"id":24072579,"defaultBranch":"master","name":"bigartm","ownerLogin":"bigartm","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2014-09-15T20:26:11.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/8783740?v=4","public":true,"private":false,"isOrgOwned":true},"symbolsExpanded":false,"treeExpanded":true,"refInfo":{"name":"master","listCacheKey":"v0:1692461886.0","canEdit":false,"refType":"branch","currentOid":"47e37f982de87aa67bfd475ff1f39da696b181b3"},"path":"docs/tutorials/python_userguide/loading_data.txt","currentUser":null,"blob":{"rawLines":["1. Loading Data: BatchVectorizer and Dictionary","===============================================","","Detailed description of all parameters and methods of BigARTM Python API classes can be found in :doc:`../../api_references/python_interface`.","","* **BatchVectorizer**:","","Before starting modeling we need to convert you data in the library format. At first you need to read about supporting formats for source data in :doc:`../datasets`. It's your task to prepare your data in one of these formats. As you had transformed your data into one of source formats, you can convert them in the BigARTM internal format (batches of documents) using ``BatchVectorizer`` class object.","","Really you have one more simple way to process your collection, if it is not too big and you don't need to store it in batches. To use it you need to archive two variables: ``numpy.ndarray`` ``n_wd`` with :math:`n_{wd}` counters and corresponding Python dict with vocabulary (key - index of ``numpy.ndarray``, value - corresponding token). The simpliest way to get these data is sklearn ``CountVectorizer`` usage (or some similar class from sklearn).","","If you have archived described variables run following code:","",".. code-block:: python",""," batch_vectorizer = artm.BatchVectorizer(data_format='bow_n_wd',"," n_wd=n_wd,"," vocabulary=vocabulary)","","If you have data in UCI format (e.g. ``vocab.my_collection.txt`` and ``docword.my_collection.txt`` files), that were put into the same directory with your script or notebook, you can create batches using the following code:","",".. code-block:: python",""," batch_vectorizer = artm.BatchVectorizer(data_path='',"," data_format='bow_uci',"," collection_name='my_collection',"," target_folder='my_collection_batches')","","The built-in library parser converted your data into batches and covered them with the ``BatchVectorizer`` class object, that is a general input data type for all methods of Python API. The batches were places in the directory, you specified in the ``target_folder`` parameter.","","If you have the source file in the Vowpal Wabbit data format, you can use the following command:","",".. code-block:: python",""," batch_vectorizer = artm.BatchVectorizer(data_path='',"," data_format='vowpal_wabbit',"," target_folder='my_collection_batches')","","The result is fully the same, as it was described above.","",".. note::",""," If you had created batches ones, you shouldn't launch this process any more, because it spends many time while dealing with large collection. You can run the following code instead. It will create the ``BatchVectorizer`` object using the existing batches (this operation is very quick):",""," .. code-block:: python"," ","\tbatch_vectorizer = artm.BatchVectorizer(data_path='my_collection_batches',"," data_format='batches')","","* **Dictionary**:","","The next step is to create ``Dictionary``. This is a data structure containing the information about all unique tokens in the collection. The dictionary is generating outside the model, and this operation can be done in different ways (load, create, gather). The most basic case is to gather dictionary using batches directory. You need to do this operation only once when starting working with new collection. Use the following code:","",".. code-block:: python",""," dictionary = artm.Dictionary()"," dictionary.gather(data_path='my_collection_batches')","","In this case the token order in the dictionary (and in further :math:`\\Phi` matrix) will be random. If you'd like to specify some order, you need to create the vocab file (see UCI format), containing all unique tokens of the collection in necessary order, and run the code below (assuming your file has ``vocab.txt`` name and located in the same directory with your code):","",".. code-block:: python",""," dictionary = artm.Dictionary()"," dictionary.gather(data_path='my_collection_batches',"," vocab_file_path='vocab.txt')","","Take into consideration the fact that library will ignore any token from batches, that was not presented into vocab file, if you used it. ``Dictionary`` contains a lot of useful information about the collection. For example, each unique token in it has the corresponding variable - value. When BigARTM gathers the dictionary, it puts the relative frequency of this token in this variable. You can read about the use-cases of this variable in further sections.","","Now you have a dictionary. It can be saved on the disk to prevent it's re-creation. You can save it in the binary format:","",".. code-block:: python",""," dictionary.save(dictionary_path='my_collection_batches/my_dictionary')","","Or in the textual one (if you'd like to see the gathered data, for example):","",".. code-block:: python",""," dictionary.save_text(dictionary_path='my_collection_batches/my_dictionary.txt')","","Saved dictionary can be loaded back. The code for binary file looks as following:","",".. code-block:: python",""," dictionary.load(dictionary_path='my_collection_batches/my_dictionary.dict')","","For textual dictionary you can run the following code:","",".. code-block:: python",""," dictionary.load_text(dictionary_path='my_collection_batches/my_dictionary.txt')","","Besides looking the content of the textual dictionary, you also can moderate it (for example, change the value of value field). After you load the dictionary back, these changes will be used.","",".. note::",""," All described ways of generating batches automatically generate dictionary. You can use it by typing:",""," .. code-block:: python",""," batch_vectorizer.dictionary",""," If you don't want to create this dictionary, set ``gather_dictionary`` parameter in the constructor of ``BatchVectorizer`` to False. But this flag will be ignored if ``data_format`` == ``bow_n_wd``, as it is the only possible way to generate dictionary in this case."],"stylingDirectives":[[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]],"colorizedLines":null,"csv":null,"csvError":null,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/bigartm/bigartm/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"loading_data.txt","displayUrl":"https://github.com/bigartm/bigartm/blob/master/docs/tutorials/python_userguide/loading_data.txt?raw=true","headerInfo":{"blobSize":"5.91 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"ab39d84","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Fbigartm%2Fbigartm%2Fblob%2Fmaster%2Fdocs%2Ftutorials%2Fpython_userguide%2Floading_data.txt","isCSV":false,"isRichtext":false,"toc":null,"lineInfo":{"truncatedLoc":"103","truncatedSloc":"58"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"issueTemplate":null,"discussionTemplate":null,"language":"Text","languageID":372,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/bigartm/bigartm/blob/master/docs/tutorials/python_userguide/loading_data.txt","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/bigartm/bigartm/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/bigartm/bigartm/raw/master/docs/tutorials/python_userguide/loading_data.txt","renderImageOrRaw":false,"richText":null,"renderedFileInfo":null,"shortPath":null,"symbolsEnabled":true,"tabSize":8,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null,"symbols":{"timed_out":false,"not_analyzed":true,"symbols":[]}},"copilotInfo":null,"copilotAccessAllowed":false,"csrf_tokens":{"/bigartm/bigartm/branches":{"post":"mgWp9dEBSgVEPGBZ4cDlY1EBlr4D0ppjejsi5Pn4ql9xcZhDJMiz4dUy2U3XuX_nZKHDI_Mdx15NaF4lc1AfdA"},"/repos/preferences":{"post":"FyC0p7LDnrkz2pCBSLRF_-A2m6gutxKsQkkp8etYlfLBC4MhKgVfQlPUq3slklltztXXiCqzbAlkDoo2cwTKLg"}}},"title":"bigartm/docs/tutorials/python_userguide/loading_data.txt at master ยท bigartm/bigartm"}