Merge pull request #33 from sergiocorreia/groupreg

Update master branch to ftools 2.48.0 to support reghdfe v6

src/ftools.ado

@@ -1,4 +1,4 @@
-*! version 2.42.0 28dec2020
+*! version 2.48.0 29mar2021
 * This file is just used to compile ftools.mlib
 
 program define ftools
@@ -15,7 +15,8 @@ program define ftools
 					  assert_msg() assert_in() assert_boolean() _assert_abort() /// bin_order()
 					  aggregate_*() select_nm_*() rowproduct() ///
 					  create_mask() update_mask() is_rowvector() clip() inrange() ///
-					  varlist_is_hybrid() varlist_is_integers()
+					  varlist_is_hybrid() varlist_is_integers() ///
+					  unlink_folder()
 		ms_compile_mata, package(ftools) version(`package_version') `force' fun(`functions') verbose // debug
 	}
 	else if "`options'"=="version" {

src/ftools.pkg

@@ -82,4 +82,8 @@ f ms_add_comma.ado
 f ms_fvstrip.ado
 f ms_fvstrip.sthlp
 
+f parallel_map.ado
+f parallel_map.sthlp
+f parallel_map_template.do.ado
+
 f ftab.ado

src/ftools_common.mata

@@ -79,7 +79,7 @@ mata:
 
 `Matrix' inrange(`Matrix' x, `Matrix' lb, `Matrix' ub)
 {
-	return(lb :< x :& x :< ub)
+	return(lb :<= x :& x :<= ub)
 }
 
 
@@ -195,4 +195,32 @@ mata:
 }
 
 
+
+`Void' unlink_folder(`String' path, `Boolean' verbose)
+{
+	// We are SUPER careful in only removing certain files... so if there are other files this function will fail
+	`StringVector'			fns, patterns
+	`Integer'				i, j, num_dropped
+	
+	if (!direxists(path)) exit()
+	if (verbose) printf("{txt}Removing folder and its contents: {res}%s{txt}\n", path)
+	
+	num_dropped = 0
+	patterns = ("*.tmp" \ "*.log" \ "parallel_code.do")
+
+	for (j=1; j<=rows(patterns); j++){
+		fns = dir(path, "files", patterns[j], 1)
+		for (i=1; i<=rows(fns); i++) {
+			unlink(fns[i])
+			++num_dropped
+		}
+	}
+
+	if (verbose) printf("{txt} - %f files removed\n", num_dropped)
+	
+	rmdir(path)
+	if (verbose) printf("{txt} - Folder removed\n")
+}
+
+
 end

src/ftools_main.mata

@@ -20,9 +20,11 @@ class Factor
 	`Boolean'				is_sorted			// Is varlist==sorted(varlist)?
 	`StringRowVector'		sortedby			// undocumented; save sort order of dataset
 	`Boolean'				panel_is_setup
+	`Boolean'				levels_as_keys		// when running F3=join_factors(F1, F2), use the levels of F1/F2 as keys for F3 (useful when F1.keys is missing)
 
 	`Void'					new()
-	`Void'					panelsetup()		// aux. vectors
+	`Void'					swap()
+	virtual `Void'			panelsetup()		// aux. vectors
 	`Void'					store_levels()		// Store levels in the dta
 	`Void'					store_keys()		// Store keys & format/lbls
 	`DataFrame'				sort()				// Initialize panel view
@@ -33,37 +35,17 @@ class Factor
 	`Boolean'				equals()			// True if F1 == F2
 
 	`Void'					__inner_drop()		// Adjust to dropping obs.
-	`Vector'				drop_singletons()	// Adjust to dropping obs.
-	`Void'					drop_obs()			// Adjust to dropping obs.
+	virtual `Vector'		drop_singletons()	// Adjust to dropping obs.
+	virtual `Void'			drop_obs()			// Adjust to dropping obs.
 	`Void'					keep_obs()			// Adjust to dropping obs.
 	`Void'					drop_if()			// Adjust to dropping obs.
 	`Void'					keep_if()			// Adjust to dropping obs.
 	`Boolean'				is_id()				// 1 if all(F.counts:==1)
 
 	`Vector'				intersect()			// 1 if Y intersects with F.keys
-	
-	// Undocumented
-	`Dict'					extra				// extra information
-
-	// Undocumented for reghdfe / groupreg
-	`String' 				absvar				// expression: "firm#year", "i.firm##c.(x1 x2)", etc.
-	`Varlist' 				ivars				// variables used for intercepts
-	`Varlist' 				cvars				// variables used for slopes
-	`Boolean'				has_intercept		// 1 for "firm" or "firm##c.year" ; 0 for "firm#c.year"
-	`Integer' 				num_slopes			// number of slopes
-	`String' 				target				// where the FE will be saved
-	`Boolean'				save_fe				// 1 if we save the FE
-
-	// Undocumented objects used for slope variables (cvars) in reghdfe/groupreg
-	`Matrix'				x					// standardized slope variables "x1 x2.."
-	`RowVector'				x_means				// means of slope variables
-	`RowVector'				x_stdevs			// standard deviations of slope variables
-	`Matrix'				inv_xx				// inv(x'x) for each fixed effect
-	
-	`Boolean'				has_weights
-	`Vector'				weights
-	`Vector'				weighted_counts
-	
+	virtual `Void'			cleanup_before_saving() // set .vl and .extra to missing
+
+	`Dict'					extra				// keep for compatibility with reghdfe v5
 }
 
 
@@ -77,8 +59,32 @@ class Factor
 	inv_p = J(0, 1, .)
 	touse = ""
 	panel_is_setup = 0
-	extra = asarray_create("string", 1, 20)
 	is_sorted = 0
+	extra = asarray_create("string", 1, 20) // keep for compatibility with reghdfe v5
+}
+
+
+`Void' Factor::swap(`Factor' other)
+{
+	::swap(this.num_levels, other.num_levels)
+	::swap(this.num_obs, other.num_obs)
+	::swap(this.touse, other.touse)
+	::swap(this.varlist, other.varlist)
+	::swap(this.varformats, other.varformats)
+	::swap(this.varlabels, other.varlabels)
+	::swap(this.varvaluelabels, other.varvaluelabels)
+	::swap(this.vartypes, other.vartypes)
+	::swap(this.vl, other.vl)
+	::swap(this.levels, other.levels)
+	::swap(this.keys, other.keys)
+	::swap(this.counts, other.counts)
+	::swap(this.info, other.info)
+	::swap(this.p, other.p)
+	::swap(this.inv_p, other.inv_p)
+	::swap(this.method, other.method)
+	::swap(this.is_sorted, other.is_sorted)
+	::swap(this.sortedby, other.sortedby)
+	::swap(this.panel_is_setup, other.panel_is_setup)
 }
 
 
@@ -353,6 +359,7 @@ class Factor
 }
 
 
+// KEPT ONLY FOR BACKWARDS COMPAT
 `Vector' Factor::drop_singletons(| `Vector' fweight,
                                    `Boolean' zero_threshold)
 {
@@ -440,6 +447,12 @@ class Factor
 }
 
 
+`Void' Factor::cleanup_before_saving()
+{
+	this.vl = this.extra = .
+}
+
+
 // Main functions -------------------------------------------------------------
 
 `Factor' factor(`Varlist' varnames,
@@ -549,7 +562,7 @@ class Factor
                  `Varlist' vars, 			// hack
                  `DataCol' touse)		 	// hack
 {
-	`Factor'				F, F1, F2
+	`Factor'				F
 	`Integer'				num_obs, num_vars
 	`Integer'				i
 	`Integer'				limit0
@@ -623,10 +636,7 @@ class Factor
 	// Hack: alternative approach
 	// all(delta :< num_obs) --> otherwise we should just run hash1
 	if (base_method == "mata" & method == "hash1" & integers_only & num_vars > 1 & cols(vars)==num_vars & num_obs > 1e5 & all(delta :< num_obs)) {
-		F1 = factor(vars[1], touse, verbose, "mata", sort_levels, 1, ., save_keys)
-		F2 = factor(vars[2..num_vars], touse, verbose, "mata", sort_levels, count_levels, ., save_keys)
-		F = join_factors(F1, F2, count_levels, save_keys)
-		F1 = F2 = Factor() // clear
+		F = _factor_alt(vars[1], vars[2..num_vars], touse, verbose, sort_levels, count_levels, save_keys)
 		method = "join"
 	}
 	else if (method == "hash0") {
@@ -659,6 +669,22 @@ class Factor
 }
 
 
+`Factor' _factor_alt(`Varname' first_var,
+					 `Varlist' other_vars,
+					 `DataCol' touse,
+					 `Boolean' verbose,
+					 `Boolean' sort_levels,
+					 `Boolean' count_levels,
+					 `Boolean' save_keys)
+{
+	`Factor'				F, F1, F2
+	F1 = factor(first_var, touse, verbose, "mata", sort_levels, 1, ., save_keys)
+	F2 = factor(other_vars, touse, verbose, "mata", sort_levels, count_levels, ., save_keys)
+	F = join_factors(F1, F2, count_levels, save_keys)
+	return(F)
+}
+
+
 `Factor' join_factors(`Factor' F1,
                       `Factor' F2, 
                     | `Boolean' count_levels,
@@ -802,7 +828,7 @@ class Factor
     F.method = "join"
     F.sortedby = F1.sortedby
     F.varlist = vars
-    if (levels_as_keys) asarray(F.extra, "levels_as_keys", 1)
+    F.levels_as_keys = levels_as_keys
 
     if (!is_sorted) levels = F1.invsort(levels)
     if (count_levels) counts = counts[| 1 \ num_levels |]

src/join.ado

@@ -1,4 +1,4 @@
-*! version 2.36.1 13feb2019
+*! version 2.48.0 29mar2021
 program define join
 
 // Parse --------------------------------------------------------------------
@@ -121,6 +121,11 @@ program define join
 	if ("`keep_nums'" == "1") qui drop if inlist(`generate', 3)
 	if ("`keep_nums'" == "3") qui drop if inlist(`generate', 1)
 
+	* Adding data should clear the sort order of the master dataset
+	if (`keep_using') {
+		ClearSortOrder
+	}
+
 	if (`report') {
 		Table `generate'
 	}
@@ -209,6 +214,31 @@ program define ParseBy
 end
 
 
+program define ClearSortOrder
+	* Andrew Maurer's trick to clear `: sortedby'
+	* copied from fsort.ado
+	* see https://github.com/sergiocorreia/ftools/issues/32
+
+	loc sortvar : sortedby
+	if ("`sortvar'" != "") {
+		loc sortvar : word 1 of `sortvar'
+		loc sortvar_type : type `sortvar'
+		loc sortvar_is_str = strpos("`sortvar_type'", "str") == 1
+		loc val = `sortvar'[1]
+
+		if (`sortvar_is_str') {
+			qui replace `sortvar' = cond(mi(`"`val'"'), ".", "") in 1
+			qui replace `sortvar' = `"`val'"' in 1
+		}
+		else {
+			qui replace `sortvar' = cond(mi(`val'), 0, .) in 1
+			qui replace `sortvar' = `val' in 1
+		}
+		assert "`: sortedby'" == ""
+	}
+end
+
+
 program define Table
 	syntax varname
 

src/ms_expand_varlist.ado

@@ -1,6 +1,6 @@
-*! version 2.30.1 18jul2018
+*! version 2.47.0 17mar2021
 program ms_expand_varlist, rclass
-	syntax [varlist(ts fv numeric default=none)] if
+	syntax [varlist(ts fv numeric default=none)] [if]
 	fvexpand `varlist' `if'
 	loc varlist  `r(varlist)'`'
 
@@ -15,18 +15,21 @@ program ms_expand_varlist, rclass
 			// gen id = _n
 			// mata: st_data(., "1.id 2.id") // The 1st column is empty!
 			// mata: st_data(., "1bn.id 2.id 3.id") // correct result
-			AddBN `part'
 			//di as error "AFTER=[`part']"
+			AddBN `part'
+			loc all_vars `all_vars' `part'
 			loc selected_vars `selected_vars' `part'
 		}
-		//else {
-		//	di as error "OMITTED/BASE: `part'"
-		//}
+		else {
+			loc all_vars `all_vars' `part'
+			*di as error "OMITTED/BASE: `part'"
+		}
 	}
 
-	return local fullvarlist	`varlist'
-	return local varlist		`selected_vars'
-	return local not_omitted	`mask'
+	return local fullvarlist	`varlist'			// i.rep78 -> 1b.rep78   2.rep78   3.rep78
+	return local fullvarlist_bn	`all_vars'			// i.rep78 -> 1b.rep78 2bn.rep78 3bn.rep78
+	return local varlist		`selected_vars'		// i.rep78 ->          2bn.rep78 3bn.rep78
+	return local not_omitted	`mask'				// i.rep78 ->    0        1			1
 end
 
 capture program drop AddBN