{"payload":{"allShortcutsEnabled":false,"fileTree":{"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss":{"items":[{"name":"JindoFsDemo.scala","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/JindoFsDemo.scala","contentType":"file"},{"name":"SparkUnstructuredDataCompute.scala","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala","contentType":"file"}],"totalCount":2},"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples":{"items":[{"name":"graphx","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/graphx","contentType":"directory"},{"name":"log4j2","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/log4j2","contentType":"directory"},{"name":"mllib","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/mllib","contentType":"directory"},{"name":"oss","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss","contentType":"directory"},{"name":"sparksql","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/sparksql","contentType":"directory"},{"name":"streaming","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/streaming","contentType":"directory"},{"name":"structuredStreaming","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/structuredStreaming","contentType":"directory"},{"name":"zeppelin","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/zeppelin","contentType":"directory"},{"name":"SparkPi.scala","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/SparkPi.scala","contentType":"file"},{"name":"WordCount.scala","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/WordCount.scala","contentType":"file"}],"totalCount":10},"spark-2.x/src/main/scala/com/aliyun/odps/spark":{"items":[{"name":"examples","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples","contentType":"directory"}],"totalCount":1},"spark-2.x/src/main/scala/com/aliyun/odps":{"items":[{"name":"spark","path":"spark-2.x/src/main/scala/com/aliyun/odps/spark","contentType":"directory"}],"totalCount":1},"spark-2.x/src/main/scala/com/aliyun":{"items":[{"name":"odps","path":"spark-2.x/src/main/scala/com/aliyun/odps","contentType":"directory"}],"totalCount":1},"spark-2.x/src/main/scala/com":{"items":[{"name":"aliyun","path":"spark-2.x/src/main/scala/com/aliyun","contentType":"directory"}],"totalCount":1},"spark-2.x/src/main/scala":{"items":[{"name":"com","path":"spark-2.x/src/main/scala/com","contentType":"directory"}],"totalCount":1},"spark-2.x/src/main":{"items":[{"name":"java","path":"spark-2.x/src/main/java","contentType":"directory"},{"name":"python","path":"spark-2.x/src/main/python","contentType":"directory"},{"name":"scala","path":"spark-2.x/src/main/scala","contentType":"directory"}],"totalCount":3},"spark-2.x/src":{"items":[{"name":"main","path":"spark-2.x/src/main","contentType":"directory"}],"totalCount":1},"spark-2.x":{"items":[{"name":"libs","path":"spark-2.x/libs","contentType":"directory"},{"name":"src","path":"spark-2.x/src","contentType":"directory"},{"name":"pom.xml","path":"spark-2.x/pom.xml","contentType":"file"}],"totalCount":3},"":{"items":[{"name":"hook","path":"hook","contentType":"directory"},{"name":"spark-1.x","path":"spark-1.x","contentType":"directory"},{"name":"spark-2.x","path":"spark-2.x","contentType":"directory"},{"name":"spark-3.x","path":"spark-3.x","contentType":"directory"},{"name":"spark-utils","path":"spark-utils","contentType":"directory"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":"LICENSE","path":"LICENSE","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"}],"totalCount":8}},"fileTreeProcessingTime":19.631966000000002,"foldersToFetch":[],"repo":{"id":187778605,"defaultBranch":"master","name":"MaxCompute-Spark","ownerLogin":"aliyun","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2019-05-21T06:51:29.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/941070?v=4","public":true,"private":false,"isOrgOwned":true},"symbolsExpanded":false,"treeExpanded":true,"refInfo":{"name":"master","listCacheKey":"v0:1682110014.0","canEdit":false,"refType":"branch","currentOid":"42410720b68384cf3e505c7cb06764f96b4ebd8a"},"path":"spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala","currentUser":null,"blob":{"rawLines":["/**"," * Licensed to the Apache Software Foundation (ASF) under one"," * or more contributor license agreements. See the NOTICE file"," * distributed with this work for additional information"," * regarding copyright ownership. The ASF licenses this file"," * to you under the Apache License, Version 2.0 (the"," * \"License\"); you may not use this file except in compliance"," * with the License. You may obtain a copy of the License at"," *
"," * http://www.apache.org/licenses/LICENSE-2.0"," *
"," * Unless required by applicable law or agreed to in writing, software"," * distributed under the License is distributed on an \"AS IS\" BASIS,"," * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied."," * See the License for the specific language governing permissions and"," * limitations under the License."," */","","package com.aliyun.odps.spark.examples.oss","","import org.apache.spark.sql.SparkSession","","object SparkUnstructuredDataCompute {"," def main(args: Array[String]) {"," val spark = SparkSession"," .builder()"," .config(\"spark.hadoop.fs.oss.credentials.provider\", \"org.apache.hadoop.fs.aliyun.oss.AliyunStsTokenCredentialsProvider\")"," .config(\"spark.hadoop.fs.oss.ststoken.roleArn\", \"acs:ram::****:role/aliyunodpsdefaultrole\")"," .config(\"spark.hadoop.fs.oss.endpoint\", \"oss-cn-hangzhou-zmf.aliyuncs.com\")"," .appName(\"SparkUnstructuredDataCompute\")"," .getOrCreate()",""," val sc = spark.sparkContext"," try {"," val pathIn = \"oss://bucket/inputdata/\""," val inputData = sc.textFile(pathIn, 5)"," val cnt = inputData.count"," println(s\"count: $cnt\")"," } finally {"," sc.stop()"," }"," }","}"],"stylingDirectives":[[{"start":0,"end":3,"cssClass":"pl-c"},{"start":0,"end":3,"cssClass":"pl-c"}],[{"start":0,"end":62,"cssClass":"pl-c"}],[{"start":0,"end":64,"cssClass":"pl-c"}],[{"start":0,"end":57,"cssClass":"pl-c"}],[{"start":0,"end":62,"cssClass":"pl-c"}],[{"start":0,"end":53,"cssClass":"pl-c"}],[{"start":0,"end":62,"cssClass":"pl-c"}],[{"start":0,"end":62,"cssClass":"pl-c"}],[{"start":0,"end":7,"cssClass":"pl-c"}],[{"start":0,"end":46,"cssClass":"pl-c"}],[{"start":0,"end":7,"cssClass":"pl-c"}],[{"start":0,"end":71,"cssClass":"pl-c"}],[{"start":0,"end":69,"cssClass":"pl-c"}],[{"start":0,"end":76,"cssClass":"pl-c"}],[{"start":0,"end":71,"cssClass":"pl-c"}],[{"start":0,"end":34,"cssClass":"pl-c"}],[{"start":0,"end":4,"cssClass":"pl-c"},{"start":2,"end":4,"cssClass":"pl-c"}],[],[{"start":0,"end":7,"cssClass":"pl-k"},{"start":8,"end":11,"cssClass":"pl-en"},{"start":12,"end":18,"cssClass":"pl-en"},{"start":19,"end":23,"cssClass":"pl-en"},{"start":24,"end":29,"cssClass":"pl-en"},{"start":30,"end":38,"cssClass":"pl-en"},{"start":39,"end":42,"cssClass":"pl-en"}],[],[{"start":0,"end":6,"cssClass":"pl-k"},{"start":7,"end":10,"cssClass":"pl-en"},{"start":11,"end":17,"cssClass":"pl-en"},{"start":18,"end":23,"cssClass":"pl-en"},{"start":24,"end":27,"cssClass":"pl-en"},{"start":28,"end":40,"cssClass":"pl-en"}],[],[{"start":0,"end":6,"cssClass":"pl-k"},{"start":7,"end":35,"cssClass":"pl-en"}],[{"start":2,"end":5,"cssClass":"pl-k"},{"start":6,"end":10,"cssClass":"pl-en"},{"start":11,"end":15,"cssClass":"pl-v"},{"start":17,"end":22,"cssClass":"pl-en"},{"start":23,"end":29,"cssClass":"pl-en"}],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":13,"cssClass":"pl-v"},{"start":14,"end":15,"cssClass":"pl-k"},{"start":16,"end":28,"cssClass":"pl-en"}],[],[{"start":14,"end":56,"cssClass":"pl-s"},{"start":14,"end":15,"cssClass":"pl-pds"},{"start":55,"end":56,"cssClass":"pl-pds"},{"start":58,"end":125,"cssClass":"pl-s"},{"start":58,"end":59,"cssClass":"pl-pds"},{"start":124,"end":125,"cssClass":"pl-pds"}],[{"start":14,"end":52,"cssClass":"pl-s"},{"start":14,"end":15,"cssClass":"pl-pds"},{"start":51,"end":52,"cssClass":"pl-pds"},{"start":54,"end":96,"cssClass":"pl-s"},{"start":54,"end":55,"cssClass":"pl-pds"},{"start":95,"end":96,"cssClass":"pl-pds"}],[{"start":14,"end":44,"cssClass":"pl-s"},{"start":14,"end":15,"cssClass":"pl-pds"},{"start":43,"end":44,"cssClass":"pl-pds"},{"start":46,"end":80,"cssClass":"pl-s"},{"start":46,"end":47,"cssClass":"pl-pds"},{"start":79,"end":80,"cssClass":"pl-pds"}],[{"start":15,"end":45,"cssClass":"pl-s"},{"start":15,"end":16,"cssClass":"pl-pds"},{"start":44,"end":45,"cssClass":"pl-pds"}],[],[],[{"start":4,"end":7,"cssClass":"pl-k"},{"start":8,"end":10,"cssClass":"pl-v"},{"start":11,"end":12,"cssClass":"pl-k"}],[{"start":4,"end":7,"cssClass":"pl-k"}],[{"start":6,"end":9,"cssClass":"pl-k"},{"start":10,"end":16,"cssClass":"pl-v"},{"start":17,"end":18,"cssClass":"pl-k"},{"start":19,"end":44,"cssClass":"pl-s"},{"start":19,"end":20,"cssClass":"pl-pds"},{"start":43,"end":44,"cssClass":"pl-pds"}],[{"start":6,"end":9,"cssClass":"pl-k"},{"start":10,"end":19,"cssClass":"pl-v"},{"start":20,"end":21,"cssClass":"pl-k"},{"start":42,"end":43,"cssClass":"pl-c1"}],[{"start":6,"end":9,"cssClass":"pl-k"},{"start":10,"end":13,"cssClass":"pl-v"},{"start":14,"end":15,"cssClass":"pl-k"}],[{"start":14,"end":15,"cssClass":"pl-k"},{"start":15,"end":16,"cssClass":"pl-s"},{"start":16,"end":23,"cssClass":"pl-s"},{"start":27,"end":28,"cssClass":"pl-s"}],[{"start":6,"end":13,"cssClass":"pl-k"}],[],[],[],[]],"colorizedLines":null,"csv":null,"csvError":null,"dependabotInfo":{"showConfigurationBanner":false,"configFilePath":null,"networkDependabotPath":"/aliyun/MaxCompute-Spark/network/updates","dismissConfigurationNoticePath":"/settings/dismiss-notice/dependabot_configuration_notice","configurationNoticeDismissed":null},"displayName":"SparkUnstructuredDataCompute.scala","displayUrl":"https://github.com/aliyun/MaxCompute-Spark/blob/master/spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala?raw=true","headerInfo":{"blobSize":"1.61 KB","deleteTooltip":"You must be signed in to make or propose changes","editTooltip":"You must be signed in to make or propose changes","ghDesktopPath":"https://desktop.github.com","isGitLfs":false,"onBranch":true,"shortPath":"636083b","siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Faliyun%2FMaxCompute-Spark%2Fblob%2Fmaster%2Fspark-2.x%2Fsrc%2Fmain%2Fscala%2Fcom%2Faliyun%2Fodps%2Fspark%2Fexamples%2Foss%2FSparkUnstructuredDataCompute.scala","isCSV":false,"isRichtext":false,"toc":null,"lineInfo":{"truncatedLoc":"43","truncatedSloc":"39"},"mode":"file"},"image":false,"isCodeownersFile":null,"isPlain":false,"isValidLegacyIssueTemplate":false,"issueTemplate":null,"discussionTemplate":null,"language":"Scala","languageID":341,"large":false,"planSupportInfo":{"repoIsFork":null,"repoOwnedByCurrentUser":null,"requestFullPath":"/aliyun/MaxCompute-Spark/blob/master/spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala","showFreeOrgGatedFeatureMessage":null,"showPlanSupportBanner":null,"upgradeDataAttributes":null,"upgradePath":null},"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_dockerfile","releasePath":"/aliyun/MaxCompute-Spark/releases/new?marketplace=true","showPublishActionBanner":false},"rawBlobUrl":"https://github.com/aliyun/MaxCompute-Spark/raw/master/spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala","renderImageOrRaw":false,"richText":null,"renderedFileInfo":null,"shortPath":null,"symbolsEnabled":true,"tabSize":8,"topBannersInfo":{"overridingGlobalFundingFile":false,"globalPreferredFundingPath":null,"showInvalidCitationWarning":false,"citationHelpUrl":"https://docs.github.com/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-citation-files","actionsOnboardingTip":null},"truncated":false,"viewable":true,"workflowRedirectUrl":null,"symbols":{"timed_out":false,"not_analyzed":false,"symbols":[{"name":"com.aliyun.odps.spark.examples.oss","kind":"module","ident_start":836,"ident_end":870,"extent_start":828,"extent_end":870,"fully_qualified_name":"com.aliyun.odps.spark.examples.oss","ident_utf16":{"start":{"line_number":18,"utf16_col":8},"end":{"line_number":18,"utf16_col":42}},"extent_utf16":{"start":{"line_number":18,"utf16_col":0},"end":{"line_number":18,"utf16_col":42}}},{"name":"main","kind":"function","ident_start":958,"ident_end":962,"extent_start":954,"extent_end":1643,"fully_qualified_name":"main","ident_utf16":{"start":{"line_number":23,"utf16_col":6},"end":{"line_number":23,"utf16_col":10}},"extent_utf16":{"start":{"line_number":23,"utf16_col":2},"end":{"line_number":41,"utf16_col":3}}},{"name":"spark","kind":"field","ident_start":994,"ident_end":999,"extent_start":954,"extent_end":1643,"fully_qualified_name":"spark","ident_utf16":{"start":{"line_number":24,"utf16_col":8},"end":{"line_number":24,"utf16_col":13}},"extent_utf16":{"start":{"line_number":23,"utf16_col":2},"end":{"line_number":41,"utf16_col":3}}},{"name":"sc","kind":"field","ident_start":1416,"ident_end":1418,"extent_start":954,"extent_end":1643,"fully_qualified_name":"sc","ident_utf16":{"start":{"line_number":32,"utf16_col":8},"end":{"line_number":32,"utf16_col":10}},"extent_utf16":{"start":{"line_number":23,"utf16_col":2},"end":{"line_number":41,"utf16_col":3}}}]}},"copilotInfo":null,"copilotAccessAllowed":false,"csrf_tokens":{"/aliyun/MaxCompute-Spark/branches":{"post":"5khIs3yojZ3KUSy0TBCOnClJ1y79WWyGRp1us-vWYt7WLuOghTdcrOoyXbHDxJkIUmqvYtY8Ov00-D-FPL8ZtA"},"/repos/preferences":{"post":"ewBNSz6FQJb9aVET83ipe0Qelf2fPHvcbiRTWggJNN1Yv8q0kguVwdpdWNDbNQ9uAbDozY3Lwde2lEm0Qe5tDA"}}},"title":"MaxCompute-Spark/spark-2.x/src/main/scala/com/aliyun/odps/spark/examples/oss/SparkUnstructuredDataCompute.scala at master ยท aliyun/MaxCompute-Spark"}