Skip to content

Commit

Permalink
Merge branch 'master' into brttnext
Browse files Browse the repository at this point in the history
  • Loading branch information
@geoffdavis
geoffdavis committed Mar 23, 2016
2 parents f2f4bd6 + 0e9b77e commit 6d2b4ee
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 22 deletions.
9 changes: 8 additions & 1 deletion bin/usarray/mk_dbops/mk_dbops.1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ or \fI-C\fP.
Perhaps you are not running a prelim orb to segregate the data. Check that you have your orbs
specified correctly.

.IP "\fINo records in certified wf db matching N25K for chan=~/HNZ|[BH]HZ/\fP"
.IP "\fINo records in certified wf db matching XXXX for chan=~/HNZ|[BH]HZ/\fP"
Check the values of \fIwfdb\fP or possibly \fIchannel_match\fP in \fImk_dbops.pf\fP. You likely are
pointing at the wrong database where the waveforms reside. Alternatively, if you are using the \fI-W\fP
option, you could be pointing at the wrong database.
Expand Down Expand Up @@ -607,6 +607,13 @@ The deployment table subset has been modified to include the vnet. This should
deal with instances for example where you had a station operational as TA, it was
adopted/transitioned to operations as N4, and then you have to close the station.

As of March 2016, the values for the deployment.time and deployment.endtime fields
will use more accurate times based on finding first or last data values from
a \fBtrloadchan(1)\fP. This helps when waveforms are written out with \fBorb2wf(1)\fP
rather than \fBorb2db(1)\fP because \fBorb2wf\fP produces a full daylong wfdisc record
rather than the more detailed time and endtime found from an \fBorb2db\fP wfdisc
record.

.SH AUTHOR
Jennifer Eakins
.br
Expand Down
4 changes: 2 additions & 2 deletions bin/usarray/mk_dbops/mk_dbops.pf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ X CEUSN/1N4
accepted_comm_types &Tbl{
DSL
WiFi
BGAN
cable
cell modem
no comms
Expand All @@ -49,6 +50,7 @@ Cal_Ore tel
DSL Frontier
DoE
Eaglecap
GCI
HPWERN
Hughes
InMarSat
Expand All @@ -63,8 +65,6 @@ Volcano Telco
Wild Blue
alltel
bell
bgan
BGAN
charter
edge
hosted
Expand Down
90 changes: 71 additions & 19 deletions bin/usarray/mk_dbops/mk_dbops.xpl
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use Datascope;
use orb ;
use utilfunct;
#use strict;
#use warnings;
use warnings;

use Getopt::Std;

Expand All @@ -14,13 +14,15 @@ use Getopt::Std;
# Add X to Adoption types, triggering write to new db for a CEUSN-type adoption/transition
# - J.Eakins 9/17/2013
#
# Switch to using trloadchan to get "true" start/end time of waveforms
# - J.Eakins 3/15/2016
#
# J.Eakins
# 1/2009
# jeakin@ucsd.edu
#

our ($opt_k,$opt_n,$opt_v,$opt_y,$opt_I,$opt_R,$opt_U,$opt_A,$opt_D,$opt_m,$opt_w,$opt_W);
our ($opt_k,$opt_v,$opt_y,$opt_I,$opt_R,$opt_U,$opt_A,$opt_D,$opt_m,$opt_w,$opt_W);
our ($opt_s,$opt_S,$opt_p,$opt_P,$opt_C,$opt_V,$opt_d,$opt_e);
our ($dbops,$snet,$sta,$mytime,$newprovider,$newcommtype,$newvnet,$mydb,$newpower,$newdutycycle,$power,$dutycycle);
our ($samecomm,$type,$atype,$stime,$table,$table_filename);
Expand All @@ -30,12 +32,14 @@ our ($nbytes,$packet,$pkttime,$srcname,$pktid,$select,$n);
our ($model,$ssident,$dltime,$idtag,$lat,$lon,$elev);
our ($pdcc,$sdcc,$dlname,$startnull,$endnull,$earntime,$atime);
our ($stasub,$dlstasub,$nullsub,$commsub,$dlsitesub,$depsub,$chansub,$stagesub,$wfsub,$vnetsub);
our ($status_orb,$cmd_orb,$prelim_orb,$wfdb,$vnet,$packet_ext);
our ($status_orb,$cmd_orb,$prelim_orb,$preorb,$wfdb,$vnet,$packet_ext);
our (%Pf,$pf,$prelimwf_db,$sitedb,$newdb,$newdbpath,$newsuffix);
our (@db,@prewf,@ops,@stage,@wfdisc,@deployment,@dlsite,@comm,@adoption);
our (@newops,@newdeployment,@newdlsite,@newcomm,@newadoption,@rm_list,$cp);
our (%adoption_types,@single_comm,@single_deployment,@single_stage,@single_dlsite,$commtype,$provider,$adoption_type);
our (@adoption_record,@dep_record,@deptable,@comm_record,@commtable,@dlsite_record);
our (@tr,$wfsta,$wfchan,$wfstr,$wfstart,$wfend,$nseg) ;


my $pgm = $0 ;
$pgm =~ s".*/"" ;
Expand All @@ -44,7 +48,7 @@ our (@adoption_record,@dep_record,@deptable,@comm_record,@commtable,@dlsite_reco

elog_notify($cmd);

if ( !getopts('knvyIRUADmw:W:s:S:p:P:C:V:d:e:') || @ARGV < 4 || @ARGV > 6 ) {
if ( !getopts('knvyIRUADm:w:W:s:S:p:P:C:V:d:e:') || @ARGV < 4 || @ARGV > 6 ) {
die ("USAGE: $0 { -I | -U | -R | -A | -D } [-k] [-y] [-v] [-V vnet] [-p pf] [-m match_pkts] [-P prelimORB] [-S statusORB] [-C cmdORB] [-w prelimwfDB] [-W wfDB] [-s siteDB] [-d yes|no|string] [-e powersource] dbopsdb snet sta timestamp [comm_provider [comm_type] ] \n");
}

Expand Down Expand Up @@ -271,9 +275,11 @@ elog_warn("\nRecords exist in dlsite table! Consider running mk_dbops with the
# not needed if you are using prelimwf_db

if ($opt_I && !$opt_w ) {
my $preorb = orbopen ($prelim_orb, "r&" );
my $preorbchk = orbopen ($prelim_orb, "r&" );
elog_notify("Prelim orb: $prelim_orb\n") if $opt_v ;
elog_die("Can't open $prelim_orb\n") if ($preorb == -1) ;
elog_die("Can't open $prelim_orb\n") if ($prelim_orb == -1) ;
orbclose($preorbchk);

}

# check for "alive" status and cmd orb if you are updating or installing.
Expand All @@ -283,19 +289,20 @@ if ($opt_U || $opt_I) {
my $statorb = orbopen ($status_orb, "r&" );
elog_notify("Status orb: $status_orb\n") if $opt_v ;
elog_die("Can't open $status_orb\n") if ($statorb == -1) ;
orbclose($statorb);

# if (!$opt_n) {
if ($opt_y) {
my $cmdorb = orbopen ($cmd_orb, "r&" );
elog_notify("Cmd orb: $cmd_orb\n") if $opt_v ;
elog_die("Can't open $cmd_orb\n") if ($cmdorb == -1) ;
orbclose($cmdorb);
}

}

if ($opt_U) {

elog_notify("Starting UPDATE specific commands\n") if $opt_v ;
elog_notify("Starting UPDATE specific commands\n") ;

elog_notify("Command line timestamp: ". &strydtime($mytime) . " used for comm.endtime, and new comm.time\n");

Expand All @@ -317,7 +324,7 @@ if ($opt_U) {
elog_complain "Update $pf\.pf with a new accepted_comm_type: $newcommtype!!\n\n";
}

@deptable = defined $newdb && ($db eq $newdb) ? @newdeployment : @deployment ;
@deptable = defined $newdb && ($dbops eq $newdb) ? @newdeployment : @deployment ;

$newpower = $opt_e && ($opt_e !~ $power) ? $opt_e : $power ;
$newdutycycle = $opt_d && ($opt_d !~ $dutycycle) ? $opt_d : $dutycycle ;
Expand Down Expand Up @@ -377,13 +384,12 @@ if ($opt_U) {

# add to new dlsite

# if (!$opt_n) {
if ($opt_y) {
elog_notify("Starting modification of dlsite via external program q330_location. \n") ;
elog_notify("This program assumes sitedb and dbops are the same..... \n") ;
&add2dlsite($newdb);
} else {
elog_notify("\n You need to run q330_location to populate dlsite table\n\n") ;
elog_notify("No modifictaions to dlsite. You need to run q330_location to populate dlsite table\n\n") ;
}

} else {
Expand Down Expand Up @@ -488,8 +494,20 @@ if ($opt_U) {
@prewf = dbsort (@prewf, "time") ; # get single record that will be updated

$prewf[3] = 0 ;
($wfsta,$wfchan,$wfstart,$wfendtime) = dbgetv(@prewf, qw( sta chan time endtime ) );
elog_notify("Verifying first sample time in wfdisc. Reviwing traces for null data fill values\n") if $opt_v ;
elog_notify( sprintf ("Looking at %s:%s - %s", $wfsta, $wfchan, strydtime($wfstart)) ) if $opt_v ;

$dtime = dbgetv(@prewf, qw( time ) );
@tr = trloadchan(@prewf, $wfstart-10, $wfendtime+10, $wfsta, $wfchan) ;
trsplit (@tr) ;
$nseg = dbquery(@tr, dbRECORD_COUNT) ;
die ( "Failed to find single trace data from $wfsta:$wfchan\n") if $nseg != 1 ;
$tr[3] = 0;
($dtime) = dbgetv(@tr, qw( time ) );

elog_notify( sprintf ("Found adjusted start time of wf: %s" , strydtime($dtime) ) ) ;

trfree(@tr);
dbfree(@prewf);

} else { # get time from prelim orb
Expand All @@ -500,29 +518,31 @@ if ($opt_U) {
$select = $dlname . "(" . $packet_ext . ")" ; # Look for data packets
}

# for some reason, I need to reopen this orb
$preorb = orbopen ($prelim_orb, "r&" );
$n = orbselect($preorb, $select) ;
$n = orbposition($preorb, "oldest") ;


if ($n < 1 ) {
elog_die("Can't select $select packets from prelim orb, $prelim_orb\n");
}

($pktid, $srcname, $pkttime, $packet, $nbytes) = orbreap($preorb) ;
$dtime = $pkttime ;

orbclose($preorb);
}

&trim($dtime) ;

&add2deployment($dbops, $vnet, $snet, $sta, $dtime, $stime, $mytime) ;

# if (!$opt_n) {
if ($opt_y) {
elog_notify("Starting modification of dlsite via external program q330_location. \n") ;
elog_notify("This program assumes sitedb and dbops are the same..... \n") ;
&add2dlsite($dbops);
} else {
elog_notify("\n You need to run q330_location to populate dlsite table\n\n") ;
elog_notify("No modifications to dlsite. You need to run q330_location to populate dlsite table\n\n") ;
}

}
Expand Down Expand Up @@ -550,14 +570,14 @@ exit ;

sub get_stageendtime { # get the stage.endtime

elog_notify ("Starting get_stageendtime\n");
elog_notify ("Starting get_stageendtime\n") if $opt_v ;

# $stagesub = "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/&&(endtime!='9999999999.99900'||endtime!>now())" ;

# if it's a "Dropped" station, endtime might be NULL/future
# get single record by sta/chan/gtype stagesub without endtime, then sort stage in reverse for most recent record?

$stagesub = $opt_D ? "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/&&(endtime=='9999999999.99900'||endtime>now())" : "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/&&(endtime!='9999999999.99900'||endtime!>now())" ;
$stagesub = $opt_D ? "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/&&(endtime=='9999999999.99900'||endtime>now())" : "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/&&(endtime!='9999999999.99900'||endtime<now())" ;
@single_stage = dbsubset (@stage, $stagesub) ;
Expand All @@ -572,6 +592,7 @@ sub get_stageendtime { # get the stage.endtime
sub get_stagestarttime { # get the stage.time
elog_notify ("Starting get_stagestarttime\n") if $opt_v ;
$stagesub = "sta=='$sta'&&$chansub&&gtype=~/sensor|seismometer/" ;
@single_stage = dbsubset (@stage, $stagesub) ;
Expand All @@ -589,20 +610,39 @@ sub get_stagestarttime { # get the stage.time
sub get_wfendtime { # get the last time of available data from DB
elog_notify ("Starting get_wfendtime\n") if $opt_v ;
@wfdisc = dbsubset (@wfdisc, $wfsub) ;
@wfdisc = dbsort (@wfdisc, "endtime") ; # get single record that will be updated
elog_die("No records in certified wf db matching $sta for $wfsub \n") if (!dbquery(@wfdisc, "dbRECORD_COUNT") ) ;
$wfdisc[3] = dbquery(@wfdisc, "dbRECORD_COUNT") - 1 ;
$wfendtime = dbgetv(@wfdisc, qw( endtime ) );
($wfsta,$wfchan,$wfstart,$wfendtime) = dbgetv(@wfdisc, qw( sta chan time endtime ) );
$wfstr = epoch2str($wfendtime,"%H:%M");
# only go into tr if endtime is "end of day" which might be safe as "23:59"??
if ($wfstr =~ /23:59/) {
elog_notify("Questionable last sample time in wfdisc. Reviwing traces for null data fill values\n") if $opt_v ;
elog_notify( sprintf ("Looking at %s:%s - %s", $wfsta, $wfchan, strydtime($wfendtime)) ) if $opt_v ;
@tr = trloadchan(@wfdisc, $wfstart-10, $wfendtime+10, $wfsta, $wfchan) ;
trsplit (@tr) ;
$nseg = dbquery(@tr, dbRECORD_COUNT) ;
die ( "Failed to find single trace data from $wfsta:$wfchan\n") if $nseg != 1 ;
$tr[3] = 0;
($wfendtime) = dbgetv(@tr, qw( endtime ) );
elog_notify( sprintf ("Found adjusted endtime of wf: %s" , strydtime($wfendtime) ) ) ;
trfree(@tr);
}
return($wfendtime);
}
sub modifydlsite { # modifydlsite($db) # modify record in dlsite table
my ($db) = @_;
elog_notify ("Starting modifydlsite\n") if $opt_v ;
@dlsite = defined $newdb && ( $db eq $newdb ) ? @newdlsite : @dlsite ;
@single_dlsite = dbsubset (@dlsite, $dlsitesub) ; # get single record that will be updated
Expand Down Expand Up @@ -671,11 +711,14 @@ sub close_deployment { # close_deployment($db, $endtime,$equip_removal,$decertt
elog_notify("Attempting subsetting based on vnet. \n") if ($opt_V) ;
@single_deployment = dbsubset (@single_deployment, $vnetsub) ;
if (dbquery(@single_deployment, "dbRECORD_COUNT") > 1) {
elog_die("Too many records in deployment table. Fix your deployment table.\n") ;
elog_die("Too many records in deployment table. Fix your deployment table.\n") ;
} elsif (dbquery(@single_deployment, "dbRECORD_COUNT") < 1) {
elog_die("Too few records in deployment table. Fix your deployment table.\n") ;
}
}
$single_deployment[3] = 0 ;
elog_notify("Attempting dbputv into single record of deployment table\n");
dbputv(@single_deployment,"endtime",$time1,"equip_remove",$time2,"decert_time",$time3) ;
Expand Down Expand Up @@ -709,6 +752,7 @@ sub modifycomm { # &modifycomm($db,$time)
my ($db,$endcommtime) = @_ ;
elog_notify ("Starting modifycomm\n") if $opt_v ;
@comm = defined $newdb && ( $db eq $newdb ) ? @newcomm : @comm ;
@single_comm = dbsubset (@comm, $commsub) ; # get single record that will be updated
Expand All @@ -733,6 +777,7 @@ sub close_comm { # &close_comm($dbops,$time)
my ($db,$endcommtime) = @_ ;
elog_notify ("Starting close_comm\n") if $opt_v ;
@comm = defined $newdb && ( $db eq $newdb ) ? @newcomm : @comm ;
@single_comm = dbsubset (@comm, $commsub) ; # get single record that will be updated
Expand Down Expand Up @@ -767,6 +812,7 @@ sub add2dlsite { # add2dlsite ($db) # add a new record to the dlsite table
sub add2comm { #add2comm($db, $sta, $time, $commtype, $provider, $power, $dutycycle) # add a new record to the comm table
my ($db, $sta, $stime, $commtype, $provider, $power, $dutycycle) = @_;
elog_notify ("Starting add2comm\n") if $opt_v ;
@comm_record = ();
push(@comm_record,
Expand Down Expand Up @@ -798,6 +844,7 @@ sub add2deployment { # ( $db, $vnet, $net, $sta, $time, $equip_install, $cert_t
my ($db, $vnet, $net, $sta, $time, $install_time, $cert_time) = @_ ;
elog_notify ("Starting add2deployment\n") if $opt_v ;
elog_notify("Db: $db; Vnet: $vnet; Snet: $net; Sta: $sta; \n") ;
@dep_record = ();
Expand Down Expand Up @@ -835,6 +882,7 @@ sub add2adoption { # $add2adoption($db,$time) # add a new record to the adopti
my ($db, $time) = @_ ;
elog_notify ("Starting add2adoption\n") if $opt_v ;
@adoption= defined $newdb && ($db eq $newdb) ? @newadoption : @adoption;
push(@adoption_record,
Expand Down Expand Up @@ -944,6 +992,10 @@ sub usage {
USAGE: $0 -R [-k] [-p pf] [-W wfDB] [-s siteDB] dbopsdb snet sta decert_time \n");

For dropping a contributing station that has not been removed:

USAGE: $0 -D [-k] [-p pf] [-W wfDB] [-s siteDB] dbopsdb snet sta decert_time \n");
For a station transition to regional network:
USAGE: $0 -A [-k] [-p pf] [-W wfDB] [-s siteDB] dbopsdb snet sta decert_time \n");
Expand Down

0 comments on commit 6d2b4ee

Please sign in to comment.