Skip to content
Permalink
Browse files
HAWQ-1802. analyze inherits resource if possible
  • Loading branch information
ztao1987 committed Aug 27, 2021
1 parent b136a8e commit fac6938054306143041be50c778202bdd6d2c26e
Show file tree
Hide file tree
Showing 18 changed files with 512 additions and 60 deletions.
@@ -1,7 +1,7 @@
------------------------------------------------------------------
-- hawq status
------------------------------------------------------------------
CREATE FUNCTION hawq_magma_status() RETURNS SETOF record LANGUAGE internal VOLATILE STRICT AS 'hawq_magma_status' WITH (OID=5085, DESCRIPTION="Return magma node information");
CREATE FUNCTION pg_catalog.hawq_magma_status() RETURNS SETOF record LANGUAGE internal VOLATILE STRICT AS 'hawq_magma_status' WITH (OID=5085, DESCRIPTION="Return magma node information");

CREATE OR REPLACE FUNCTION pg_catalog.magma_getstatus() RETURNS void
AS '$libdir/magma.so', 'magma_getstatus'
@@ -6,10 +6,10 @@
--
-- --------------------------------------------------------------------

CREATE OR REPLACE FUNCTION hive_validate() RETURNS void
CREATE OR REPLACE FUNCTION pg_catalog.hive_validate() RETURNS void
AS '$libdir/exthive.so', 'hiveprotocol_validate'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION hive_blocklocation() RETURNS void
CREATE OR REPLACE FUNCTION pg_catalog.hive_blocklocation() RETURNS void
AS '$libdir/exthive.so', 'hiveprotocol_blocklocation'
LANGUAGE C STABLE;
@@ -643,7 +643,7 @@ Datum orc_beginscan(PG_FUNCTION_ARGS)
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

/* 5. Save file_scan_desc */
@@ -772,7 +772,7 @@ Datum orc_getnext(PG_FUNCTION_ARGS) {
ORCFormatEndORCFormatC(user_data->fmt);
e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION) {
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

ORCFormatFreeORCFormatC(&(user_data->fmt));
@@ -802,7 +802,7 @@ Datum orc_getnext(PG_FUNCTION_ARGS) {
slot->PRIVATE_tts_values = NULL;
ExecClearTuple(slot);
} else {
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}
PG_RETURN_BOOL(false);
}
@@ -865,14 +865,14 @@ Datum orc_rescan(PG_FUNCTION_ARGS)
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}
} else {
ORCFormatRescanORCFormatC(user_data->fmt);
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}
}

@@ -993,7 +993,7 @@ Datum orc_stopscan(PG_FUNCTION_ARGS)
ORCFormatEndORCFormatC(user_data->fmt);
e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION) {
ereport(LOG, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(LOG, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

ORCFormatFreeORCFormatC(&(user_data->fmt));
@@ -1029,7 +1029,7 @@ Datum orc_stopscan(PG_FUNCTION_ARGS)
tts->PRIVATE_tts_isnull = NULL;
ExecClearTuple(tts);
} else {
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

PG_RETURN_VOID();
@@ -1261,7 +1261,7 @@ Datum orc_insert_init(PG_FUNCTION_ARGS)
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

/* 4. Save the result */
@@ -1515,7 +1515,7 @@ Datum orc_insert(PG_FUNCTION_ARGS)
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR,(errcode(e->errCode),errmsg("ORC::%s", e->errMessage)));
ereport(ERROR,(errcode(e->errCode),errmsg("%s", e->errMessage)));
}

ps->ps_tuple_oid = InvalidOid;
@@ -1540,7 +1540,7 @@ Datum orc_insert_finish(PG_FUNCTION_ARGS)
ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
{
ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
}

ORCFormatFreeORCFormatC(&(user_data->fmt));
@@ -60,26 +60,26 @@ CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void
AS '$libdir/orc.so', 'orc_insert_finish'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION hdfs_validate() RETURNS void
CREATE OR REPLACE FUNCTION pg_catalog.hdfs_validate() RETURNS void
AS '$libdir/exthdfs.so', 'hdfsprotocol_validate'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION hdfs_blocklocation() RETURNS void
CREATE OR REPLACE FUNCTION pg_catalog.hdfs_blocklocation() RETURNS void
AS '$libdir/exthdfs.so', 'hdfsprotocol_blocklocation'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION csv_in() RETURNS record
CREATE OR REPLACE FUNCTION pg_catalog.csv_in() RETURNS record
AS '$libdir/extfmtcsv.so', 'extfmtcsv_in'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION csv_out(record) RETURNS bytea
CREATE OR REPLACE FUNCTION pg_catalog.csv_out(record) RETURNS bytea
AS '$libdir/extfmtcsv.so', 'extfmtcsv_out'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION text_in() RETURNS record
CREATE OR REPLACE FUNCTION pg_catalog.text_in() RETURNS record
AS '$libdir/extfmtcsv.so', 'extfmttext_in'
LANGUAGE C STABLE;

CREATE OR REPLACE FUNCTION text_out(record) RETURNS bytea
CREATE OR REPLACE FUNCTION pg_catalog.text_out(record) RETURNS bytea
AS '$libdir/extfmtcsv.so', 'extfmttext_out'
LANGUAGE C STABLE;
@@ -46,16 +46,18 @@ install-local: installdirs-local
$(INSTALL_DATA) $(srcdir)/nls-global.mk '$(DESTDIR)$(pgxsdir)/$(subdir)/nls-global.mk'

installdirs: installdirs-local
ifeq ($(with_oushudb_extension), yes)
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_* $(DESTDIR)${bindir}
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat* $(DESTDIR)${bindir}
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_magma_tool $(DESTDIR)${bindir}
${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/template-magma-site.xml $(DESTDIR)${sysconfdir}
${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/magma-site.xml $(DESTDIR)${sysconfdir}
endif

installdirs-local:
$(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(subdir)'
ifeq ($(with_oushudb_extension), yes)
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_ctl '$(DESTDIR)${bindir}/magma_ctl'
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_server '$(DESTDIR)${bindir}/magma_server'
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat_monitor '$(DESTDIR)${bindir}/hawq_heartbeat_monitor'
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat_sender '$(DESTDIR)${bindir}/hawq_heartbeat_sender'
${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_magma_tool '$(DESTDIR)${bindir}/hawq_magma_tool'
${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/template-magma-site.xml '$(DESTDIR)${sysconfdir}/template-magma-site.xml'
${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/magma-site.xml '$(DESTDIR)${sysconfdir}/magma-site.xml'
endif

uninstall: uninstall-local

@@ -122,7 +122,7 @@ static void checkOrcError(OrcFormatData *orcFormatData) {
ORCFormatFreeORCFormatC(&orcFormatData->fmt);
ORCFormatFreeStorageFormatC(&orcFormatData->updateDeleteFmt);
ereport(ERROR,
(errcode(errBuf.errCode), errmsg("ORC: %s", errBuf.errMessage)));
(errcode(errBuf.errCode), errmsg("%s", errBuf.errMessage)));
}
}

@@ -144,6 +144,7 @@ install-data: $(BKIFILES) installdirs
$(INSTALL_DATA) $(call vpathsearch,filesystem.sql) '$(DESTDIR)$(datadir)/filesystem.sql'
$(INSTALL_DATA) $(srcdir)/../utils/adt/array_distance_install.sql '$(DESTDIR)$(datadir)/array_distance_install.sql'
$(INSTALL_DATA) $(srcdir)/../utils/adt/array_distance_uninstall.sql '$(DESTDIR)$(datadir)/array_distance_uninstall.sql'
$(INSTALL_DATA) $(srcdir)/../utils/adt/json_install.sql '$(DESTDIR)$(datadir)/json_install.sql'

ifeq ($(with_orc), yes)
$(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_install.sql '$(DESTDIR)$(datadir)/orc_install.sql'
@@ -970,6 +970,9 @@ relationId == PgResQueueToastIndex ||
Oid
GetNewOid(Relation relation)
{
if (gp_upgrade_mode && gen_new_oid_value != InvalidOid) {
return gen_new_oid_value;
}
Oid newOid;
Oid oidIndex;
Relation indexrel;
@@ -790,6 +790,21 @@ void CheckAttributeForOrc(TupleDesc desc) {
errmsg("array type of column \"%s\" is not supported yet",
NameStr(desc->attrs[i]->attname))));
}

int32_t datatype =
(int32_t)(((Form_pg_attribute)(desc->attrs[i]))->atttypid);
int4 typmod = ((Form_pg_attribute)(desc->attrs[i]))->atttypmod;
if (HAWQ_TYPE_NUMERIC == datatype) {
int4 tmp_typmod = typmod - VARHDRSZ;
int precision = (tmp_typmod >> 16) & 0xffff;
int scale = tmp_typmod & 0xffff;
if (precision < 1 || 38 < precision)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("ORC DECIMAL precision must be between 1 and 38")));
if (scale == 0)
ereport(NOTICE, (errmsg("Using a scale of zero for ORC DECIMAL")));
}
}
}

@@ -522,15 +522,24 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
CurrentResourceOwner = asOwner;

/**
* we use preferred_seg_num as default and
* compute target_seg_num based on data size and distributed type
* if there is no preferred_seg_num.
* We allocate query resource for analyze if there is no active resource
* otherwise, we inherit instead
*/
QueryResource *savedResource = NULL;
savedResource = GetActiveQueryResource();

QueryResource *resource = NULL;
int target_seg_num = preferred_seg_num;
if (target_seg_num <= 0) {
target_seg_num = calculate_virtual_segment_number(lFullRelOids);
if (!savedResource) {
/**
* we use preferred_seg_num as default and
* compute target_seg_num based on data size and distributed type
* if there is no preferred_seg_num.
*/
if (target_seg_num <= 0) {
target_seg_num = calculate_virtual_segment_number(lFullRelOids);
}
}
elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);

/**
* We open relations with appropreciate locks
@@ -559,13 +568,12 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
}
}

/**
* We allocate query resource for analyze
*/
QueryResource *resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
QueryResource *savedResource = NULL;

savedResource = GetActiveQueryResource();
if (!savedResource) {
elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
} else {
resource = AllocateResource(QRL_INHERIT, 0, 0, 0, 0, NULL, 0);
}
SetActiveQueryResource(resource);

/**
@@ -6,10 +6,10 @@
--
-- --------------------------------------------------------------------

CREATE OR REPLACE FUNCTION euclidean_metric_float4array(anyarray, anyarray) RETURNS float4 LANGUAGE internal IMMUTABLE AS 'euclidean_metric_float4array';
CREATE OR REPLACE FUNCTION pg_catalog.euclidean_metric_float4array(anyarray, anyarray) RETURNS float4 LANGUAGE internal IMMUTABLE AS 'euclidean_metric_float4array';

CREATE OR REPLACE FUNCTION euclidean_metric_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal STABLE AS 'euclidean_metric_float8array';
CREATE OR REPLACE FUNCTION pg_catalog.euclidean_metric_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal STABLE AS 'euclidean_metric_float8array';

CREATE OR REPLACE FUNCTION cosine_distance_float4array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float4array';
CREATE OR REPLACE FUNCTION pg_catalog.cosine_distance_float4array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float4array';

CREATE OR REPLACE FUNCTION cosine_distance_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float8array';
CREATE OR REPLACE FUNCTION pg_catalog.cosine_distance_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float8array';

0 comments on commit fac6938

Please sign in to comment.