Skip to content

Commit

Permalink
Revert "[SPARK-46437][DOCS] Remove cruft from the built-in SQL functi…
Browse files Browse the repository at this point in the history 
…ons documentation"

This reverts commit e4b5977.
  • Loading branch information
@HyukjinKwon
HyukjinKwon committed Jan 8, 2024
1 parent 3beec9a commit a88c64e
Show file tree
Hide file tree
Showing 4 changed files with 186 additions and 113 deletions.
1 change: 0 additions & 1 deletion docs/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
generated-*.html
_generated_function_html/
281 changes: 179 additions & 102 deletions docs/sql-ref-functions-builtin.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,125 +17,202 @@ license: |
limitations under the License.
---

* Table of contents
{:toc}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-agg-funcs-table.html' %}
### Aggregate Functions
{% include_relative _generated_function_html/agg-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/agg-funcs-examples.html %}

{% include_relative generated-agg-funcs-table.html %}
#### Examples
{% include_relative generated-agg-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-window-funcs-table.html' %}
### Window Functions
{% include_relative _generated_function_html/window-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/window-funcs-examples.html %}

{% include_relative generated-window-funcs-table.html %}
#### Examples
{% include_relative generated-window-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-array-funcs-table.html' %}
### Array Functions
{% include_relative _generated_function_html/array-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/array-funcs-examples.html %}

{% include_relative generated-array-funcs-table.html %}
#### Examples
{% include_relative generated-array-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-collection-funcs-table.html' %}
### Collection Functions
{% include_relative _generated_function_html/collection-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/collection-funcs-examples.html %}

{% include_relative generated-collection-funcs-table.html %}
#### Examples
{% include_relative generated-collection-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-struct-funcs-table.html' %}
### STRUCT Functions
{% include_relative _generated_function_html/struct-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/struct-funcs-examples.html %}

{% include_relative generated-struct-funcs-table.html %}
#### Examples
{% include_relative generated-struct-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-map-funcs-table.html' %}
### Map Functions
{% include_relative _generated_function_html/map-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/map-funcs-examples.html %}

{% include_relative generated-map-funcs-table.html %}
#### Examples
{% include_relative generated-map-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-datetime-funcs-table.html' %}
### Date and Timestamp Functions
{% include_relative _generated_function_html/datetime-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/datetime-funcs-examples.html %}

{% include_relative generated-datetime-funcs-table.html %}
#### Examples
{% include_relative generated-datetime-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-math-funcs-table.html' %}
### Mathematical Functions
{% include_relative _generated_function_html/math-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/math-funcs-examples.html %}

{% include_relative generated-math-funcs-table.html %}
#### Examples
{% include_relative generated-math-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-string-funcs-table.html' %}
### String Functions
{% include_relative _generated_function_html/string-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/string-funcs-examples.html %}

{% include_relative generated-string-funcs-table.html %}
#### Examples
{% include_relative generated-string-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-conditional-funcs-table.html' %}
### Conditional Functions
{% include_relative _generated_function_html/conditional-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/conditional-funcs-examples.html %}

{% include_relative generated-conditional-funcs-table.html %}
#### Examples
{% include_relative generated-conditional-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-hash-funcs-table.html' %}
### Hash Functions
{% include_relative _generated_function_html/hash-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/hash-funcs-examples.html %}

{% include_relative generated-hash-funcs-table.html %}
#### Examples
{% include_relative generated-hash-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-csv-funcs-table.html' %}
### CSV Functions
{% include_relative _generated_function_html/csv-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/csv-funcs-examples.html %}

{% include_relative generated-csv-funcs-table.html %}
#### Examples
{% include_relative generated-csv-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-json-funcs-table.html' %}
### JSON Functions
{% include_relative _generated_function_html/json-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/json-funcs-examples.html %}

{% include_relative generated-json-funcs-table.html %}
#### Examples
{% include_relative generated-json-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-xml-funcs-table.html' %}
### XML Functions
{% include_relative _generated_function_html/xml-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/xml-funcs-examples.html %}

{% include_relative generated-xml-funcs-table.html %}
#### Examples
{% include_relative generated-xml-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-url-funcs-table.html' %}
### URL Functions
{% include_relative _generated_function_html/url-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/url-funcs-examples.html %}

{% include_relative generated-url-funcs-table.html %}
#### Examples
{% include_relative generated-url-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-bitwise-funcs-table.html' %}
### Bitwise Functions
{% include_relative _generated_function_html/bitwise-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/bitwise-funcs-examples.html %}

{% include_relative generated-bitwise-funcs-table.html %}
#### Examples
{% include_relative generated-bitwise-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-conversion-funcs-table.html' %}
### Conversion Functions
{% include_relative _generated_function_html/conversion-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/conversion-funcs-examples.html %}

{% include_relative generated-conversion-funcs-table.html %}
#### Examples
{% include_relative generated-conversion-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-predicate-funcs-table.html' %}
### Predicate Functions
{% include_relative _generated_function_html/predicate-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/predicate-funcs-examples.html %}

{% include_relative generated-predicate-funcs-table.html %}
#### Examples
{% include_relative generated-predicate-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-misc-funcs-table.html' %}
### Misc Functions
{% include_relative _generated_function_html/misc-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/misc-funcs-examples.html %}

{% include_relative generated-misc-funcs-table.html %}
#### Examples
{% include_relative generated-misc-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}

{% for static_file in site.static_files %}
{% if static_file.name == 'generated-generator-funcs-table.html' %}
### Generator Functions
{% include_relative _generated_function_html/generator-funcs-table.html %}

**Examples**
{% include_relative _generated_function_html/generator-funcs-examples.html %}
{% include_relative generated-generator-funcs-table.html %}
#### Examples
{% include_relative generated-generator-funcs-examples.html %}
{% break %}
{% endif %}
{% endfor %}
2 changes: 1 addition & 1 deletion docs/sql-ref-functions.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ license: |
---

Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs).
Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/index.html) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.
Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.

### Built-in Functions

Expand Down
15 changes: 6 additions & 9 deletions sql/gen-sql-functions-docs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,16 @@
#

import itertools
import os
import re
import shutil
from collections import namedtuple
from pathlib import Path

# To avoid adding a new direct dependency, we import markdown from within mkdocs.
from mkdocs.structure.pages import markdown

from pyspark.java_gateway import launch_gateway


SPARK_PROJECT_ROOT = Path(__file__).parents[1]

ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group")

groups = {
Expand Down Expand Up @@ -213,7 +210,7 @@ def generate_functions_table_html(jvm, html_output_dir):
for key, infos in _list_grouped_function_infos(jvm):
function_table = _make_pretty_usage(infos)
key = key.replace("_", "-")
with open(html_output_dir / f"{key}-table.html", 'w') as table_html:
with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w') as table_html:
table_html.write(function_table)


Expand All @@ -236,16 +233,16 @@ def generate_functions_examples_html(jvm, jspark, html_output_dir):
examples = _make_pretty_examples(jspark, infos)
key = key.replace("_", "-")
if examples is not None:
with open(html_output_dir / f"{key}-examples.html", 'w') as examples_html:
with open("%s/generated-%s-examples.html" % (
html_output_dir, key), 'w') as examples_html:
examples_html.write(examples)


if __name__ == "__main__":
jvm = launch_gateway().jvm
jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate()
jspark.sparkContext().setLogLevel("ERROR") # Make it less noisy.
html_output_dir = SPARK_PROJECT_ROOT / "docs" / "_generated_function_html"
shutil.rmtree(html_output_dir, ignore_errors=True)
html_output_dir.mkdir()
spark_root_dir = os.path.dirname(os.path.dirname(__file__))
html_output_dir = os.path.join(spark_root_dir, "docs")
generate_functions_table_html(jvm, html_output_dir)
generate_functions_examples_html(jvm, jspark, html_output_dir)

0 comments on commit a88c64e

Please sign in to comment.