diff --git a/.gitignore b/.gitignore index da76a414865e5..201a965a0f409 100644 --- a/.gitignore +++ b/.gitignore @@ -30,15 +30,11 @@ pandas/io/*.dat pandas/io/*.json *.log .noseids - -.idea/libraries/sass_stdlib.xml - -.idea/pandas.iml .build_cache_dir .vagrant *.whl **/wheelhouse/* - .project .pydevproject .settings +.idea diff --git a/doc/source/v0.13.0.txt b/doc/source/v0.13.0.txt index 0e3c3b50fcd85..eb5ae6740044d 100644 --- a/doc/source/v0.13.0.txt +++ b/doc/source/v0.13.0.txt @@ -425,6 +425,29 @@ Enhancements the file if the data has correctly separated and properly aligned columns using the delimiter provided to the function (:issue:`4488`). + - support for nanosecond times in periods + + .. warning:: + + These operations require ``numpy >= 1.7`` + + Period conversions in the range of seconds and below were reworked and extended + up to nanoseconds. Periods in the nanosecond range are now available. + + .. ipython:: python + date_range('2013-01-01', periods=5, freq='5N') + + or with frequency as offset + + .. ipython:: python + date_range('2013-01-01', periods=5, freq=pd.offsets.Nano(5)) + + Timestamps can be modified in the nanosecond range + + .. ipython:: python + t = Timestamp('20130101 09:01:02') + t + pd.datetools.Nano(123) + .. _whatsnew_0130.experimental: Experimental diff --git a/pandas/src/period.c b/pandas/src/period.c index 4e7ab44c7b150..2e544afce9da2 100644 --- a/pandas/src/period.c +++ b/pandas/src/period.c @@ -272,6 +272,162 @@ int dInfoCalc_SetFromAbsDate(register struct date_info *dinfo, // helpers for frequency conversion routines // +static int daytime_conversion_factors[][2] = { + { FR_DAY, 1 }, + { FR_HR, 24 }, + { FR_MIN, 60 }, + { FR_SEC, 60 }, + { FR_MS, 1000 }, + { FR_US, 1000 }, + { FR_NS, 1000 }, + { 0, 0 } +}; + +static npy_int64** daytime_conversion_factor_matrix = NULL; + +static int max_value(int a, int b) { + return a > b ? a : b; +} + +static int min_value(int a, int b) { + return a < b ? a : b; +} + +static int get_freq_group(int freq) { + return (freq/1000)*1000; +} + +static int get_freq_group_index(int freq) { + return freq/1000; +} + +static int calc_conversion_factors_matrix_size() { + int matrix_size = 0; + int index; + for (index=0;; index++) { + int period_value = get_freq_group_index(daytime_conversion_factors[index][0]); + if (period_value == 0) { + break; + } + matrix_size = max_value(matrix_size, period_value); + } + return matrix_size + 1; +} + +static void alloc_conversion_factors_matrix(int matrix_size) { + int row_index; + int column_index; + daytime_conversion_factor_matrix = malloc(matrix_size * sizeof(**daytime_conversion_factor_matrix)); + for (row_index = 0; row_index < matrix_size; row_index++) { + daytime_conversion_factor_matrix[row_index] = malloc(matrix_size * sizeof(**daytime_conversion_factor_matrix)); + for (column_index = 0; column_index < matrix_size; column_index++) { + daytime_conversion_factor_matrix[row_index][column_index] = 0; + } + } +} + +static npy_int64 calculate_conversion_factor(int start_value, int end_value) { + npy_int64 conversion_factor = 0; + int index; + for (index=0;; index++) { + int freq_group = daytime_conversion_factors[index][0]; + + if (freq_group == 0) { + conversion_factor = 0; + break; + } + + if (freq_group == start_value) { + conversion_factor = 1; + } else { + conversion_factor *= daytime_conversion_factors[index][1]; + } + + if (freq_group == end_value) { + break; + } + } + return conversion_factor; +} + +static void populate_conversion_factors_matrix() { + int row_index_index; + int row_value, row_index; + int column_index_index; + int column_value, column_index; + + for (row_index_index = 0;; row_index_index++) { + row_value = daytime_conversion_factors[row_index_index][0]; + if (row_value == 0) { + break; + } + row_index = get_freq_group_index(row_value); + for (column_index_index = row_index_index;; column_index_index++) { + column_value = daytime_conversion_factors[column_index_index][0]; + if (column_value == 0) { + break; + } + column_index = get_freq_group_index(column_value); + + daytime_conversion_factor_matrix[row_index][column_index] = calculate_conversion_factor(row_value, column_value); + } + } +} + +static void initialize_daytime_conversion_factor_maxtrix() { + int matrix_size = calc_conversion_factors_matrix_size(); + alloc_conversion_factors_matrix(matrix_size); + populate_conversion_factors_matrix(); +} + +npy_int64 get_daytime_conversion_factor(int index1, int index2) +{ + if (daytime_conversion_factor_matrix == NULL) { + initialize_daytime_conversion_factor_maxtrix(); + } + return daytime_conversion_factor_matrix[min_value(index1, index2)][max_value(index1, index2)]; +} + +npy_int64 convert_daytime(npy_int64 ordinal, int from, int to, int atEnd) +{ + int from_index, to_index, offset; + npy_int64 conversion_factor; + + if (from == to) { + return ordinal; + } + + from_index = get_freq_group_index(from); + to_index = get_freq_group_index(to); + + conversion_factor = get_daytime_conversion_factor(from_index, to_index); + + offset = atEnd ? 1 : 0; + + if (from <= to) { + return (ordinal + offset) * conversion_factor - offset; + } else { + return ordinal / conversion_factor; + } + +} + +static npy_int64 transform_via_day(npy_int64 ordinal, char relation, asfreq_info *af_info, freq_conv_func first_func, freq_conv_func second_func) { + int tempStore = af_info->targetFreq; + npy_int64 result; + + af_info->targetFreq = FR_DAY; + result = (*first_func)(ordinal, relation, af_info); + af_info->targetFreq = tempStore; + + tempStore = af_info->sourceFreq; + af_info->sourceFreq = FR_DAY; + result = (*second_func)(result, relation, af_info); + af_info->sourceFreq = tempStore; + + return result; +} + static npy_int64 DtoB_weekday(npy_int64 absdate) { return (((absdate) / 7) * 5) + (absdate) % 7 - BDAY_OFFSET; } @@ -302,24 +458,23 @@ static npy_int64 absdate_from_ymd(int y, int m, int d) { //************ FROM DAILY *************** -static npy_int64 asfreq_DtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { - +static npy_int64 asfreq_DTtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; - if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + ordinal = convert_daytime(ordinal, af_info->sourceFreq, FR_DAY, 0); + if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, GREGORIAN_CALENDAR)) + return INT_ERR_CODE; if (dinfo.month > af_info->to_a_year_end) { - return (npy_int64)(dinfo.year + 1 - BASE_YEAR); - } + return (npy_int64)(dinfo.year + 1 - BASE_YEAR); + } else { - return (npy_int64)(dinfo.year - BASE_YEAR); - } + return (npy_int64)(dinfo.year - BASE_YEAR); + } } -static npy_int64 DtoQ_yq(npy_int64 ordinal, asfreq_info *af_info, - int *year, int *quarter) { +static npy_int64 DtoQ_yq(npy_int64 ordinal, asfreq_info *af_info, int *year, int *quarter) { struct date_info dinfo; - if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, GREGORIAN_CALENDAR)) + return INT_ERR_CODE; if (af_info->to_q_year_end != 12) { dinfo.month -= af_info->to_q_year_end; if (dinfo.month <= 0) { dinfo.month += 12; } @@ -333,11 +488,11 @@ static npy_int64 DtoQ_yq(npy_int64 ordinal, asfreq_info *af_info, return 0; } - -static npy_int64 asfreq_DtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { - +static npy_int64 asfreq_DTtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { int year, quarter; + ordinal = convert_daytime(ordinal, af_info->sourceFreq, FR_DAY, 0); + if (DtoQ_yq(ordinal, af_info, &year, &quarter) == INT_ERR_CODE) { return INT_ERR_CODE; } @@ -345,23 +500,28 @@ static npy_int64 asfreq_DtoQ(npy_int64 ordinal, char relation, asfreq_info *af_i return (npy_int64)((year - BASE_YEAR) * 4 + quarter - 1); } -static npy_int64 asfreq_DtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { - +static npy_int64 asfreq_DTtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; + + ordinal = convert_daytime(ordinal, af_info->sourceFreq, FR_DAY, 0); + if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, GREGORIAN_CALENDAR)) return INT_ERR_CODE; return (npy_int64)((dinfo.year - BASE_YEAR) * 12 + dinfo.month - 1); } -static npy_int64 asfreq_DtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { +static npy_int64 asfreq_DTtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + ordinal = convert_daytime(ordinal, af_info->sourceFreq, FR_DAY, 0); return (ordinal + ORD_OFFSET - (1 + af_info->to_week_end))/7 + 1 - WEEK_OFFSET; } -static npy_int64 asfreq_DtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { - +static npy_int64 asfreq_DTtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; - if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + + ordinal = convert_daytime(ordinal, af_info->sourceFreq, FR_DAY, 0); + + if (dInfoCalc_SetFromAbsDate(&dinfo, ordinal + ORD_OFFSET, GREGORIAN_CALENDAR)) + return INT_ERR_CODE; if (relation == 'S') { return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); @@ -370,184 +530,93 @@ static npy_int64 asfreq_DtoB(npy_int64 ordinal, char relation, asfreq_info *af_i } } -// needed for getDateInfo function -static npy_int64 asfreq_DtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) { return ordinal; } +// all intra day calculations are now done within one function +static npy_int64 asfreq_WithinDT(npy_int64 ordinal, char relation, asfreq_info *af_info) { + //if (relation == 'E') { + // ordinal += 1; + //} -static npy_int64 asfreq_DtoHIGHFREQ(npy_int64 ordinal, char relation, npy_int64 per_day) { - if (relation == 'S') { - return ordinal * per_day; - } - else { - return (ordinal+ 1) * per_day - 1; - } + return convert_daytime(ordinal, af_info->sourceFreq, af_info->targetFreq, relation == 'E'); } -static npy_int64 asfreq_DtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoHIGHFREQ(ordinal, relation, 24); } -static npy_int64 asfreq_DtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoHIGHFREQ(ordinal, relation, 24*60); } -static npy_int64 asfreq_DtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoHIGHFREQ(ordinal, relation, 24*60*60); } - -//************ FROM SECONDLY *************** - -static npy_int64 asfreq_StoD(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return (ordinal)/(60*60*24); } - -static npy_int64 asfreq_StoA(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoA(asfreq_StoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } - -static npy_int64 asfreq_StoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_StoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } - -static npy_int64 asfreq_StoM(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoM(asfreq_StoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } - -static npy_int64 asfreq_StoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_StoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } - -static npy_int64 asfreq_StoB(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoB(asfreq_StoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } +//************ FROM BUSINESS *************** +static npy_int64 asfreq_BtoDT(npy_int64 ordinal, char relation, asfreq_info *af_info) +{ + ordinal += BDAY_OFFSET; + ordinal = (((ordinal - 1) / 5) * 7 + + mod_compat(ordinal - 1, 5) + 1 - ORD_OFFSET); -static npy_int64 asfreq_StoT(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return ordinal / 60; + return convert_daytime(ordinal, FR_DAY, af_info->targetFreq, relation != 'S'); } -static npy_int64 asfreq_StoH(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return ordinal / (60*60); +static npy_int64 asfreq_BtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_BtoDT, asfreq_DTtoA); } -//************ FROM MINUTELY *************** - -static npy_int64 asfreq_TtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return (ordinal)/(60*24); } - -static npy_int64 asfreq_TtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoA(asfreq_TtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_TtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_TtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_TtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoM(asfreq_TtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_TtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_TtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_TtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoB(asfreq_TtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } - -static npy_int64 asfreq_TtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return ordinal / 60; +static npy_int64 asfreq_BtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_BtoDT, asfreq_DTtoQ); } -static npy_int64 asfreq_TtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) { - if (relation == 'S') { - return ordinal*60; } - else { - return ordinal*60 + 59; - } -} - -//************ FROM HOURLY *************** - -static npy_int64 asfreq_HtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return ordinal / 24; } -static npy_int64 asfreq_HtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoA(asfreq_HtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_HtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_HtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_HtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoM(asfreq_HtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_HtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_HtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } -static npy_int64 asfreq_HtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoB(asfreq_HtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } - -// calculation works out the same as TtoS, so we just call that function for HtoT -static npy_int64 asfreq_HtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_TtoS(ordinal, relation, &NULL_AF_INFO); } - -static npy_int64 asfreq_HtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) { - if (relation == 'S') { - return ordinal*60*60; - } - else { - return (ordinal + 1)*60*60 - 1; - } +static npy_int64 asfreq_BtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_BtoDT, asfreq_DTtoM); } -//************ FROM BUSINESS *************** - -static npy_int64 asfreq_BtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) - { - ordinal += BDAY_OFFSET; - return (((ordinal - 1) / 5) * 7 + - mod_compat(ordinal - 1, 5) + 1 - ORD_OFFSET); - } - -static npy_int64 asfreq_BtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoA(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } - -static npy_int64 asfreq_BtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } - -static npy_int64 asfreq_BtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoM(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } - -static npy_int64 asfreq_BtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } +static npy_int64 asfreq_BtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_BtoDT, asfreq_DTtoW); +} -static npy_int64 asfreq_BtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoH(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } +//************ FROM WEEKLY *************** -static npy_int64 asfreq_BtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoT(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } +static npy_int64 asfreq_WtoDT(npy_int64 ordinal, char relation, asfreq_info *af_info) { + ordinal += WEEK_OFFSET; + if (relation != 'S') { + ordinal += 1; + } -static npy_int64 asfreq_BtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoS(asfreq_BtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } + ordinal = ordinal * 7 - 6 + af_info->from_week_end - ORD_OFFSET; -//************ FROM WEEKLY *************** + if (relation != 'S') { + ordinal -= 1; + } -static npy_int64 asfreq_WtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) { - ordinal += WEEK_OFFSET; - if (relation == 'S') { - return ordinal * 7 - 6 + af_info->from_week_end - ORD_OFFSET; - } - else { - return ordinal * 7 + af_info->from_week_end - ORD_OFFSET; - } + return convert_daytime(ordinal, FR_DAY, af_info->targetFreq, relation != 'S'); } static npy_int64 asfreq_WtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoA(asfreq_WtoD(ordinal, 'E', af_info), relation, af_info); } + return transform_via_day(ordinal, relation, af_info, asfreq_WtoDT, asfreq_DTtoA); +} + static npy_int64 asfreq_WtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoQ(asfreq_WtoD(ordinal, 'E', af_info), relation, af_info); } + return transform_via_day(ordinal, relation, af_info, asfreq_WtoDT, asfreq_DTtoQ); +} + static npy_int64 asfreq_WtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoM(asfreq_WtoD(ordinal, 'E', af_info), relation, &NULL_AF_INFO); } + return transform_via_day(ordinal, relation, af_info, asfreq_WtoDT, asfreq_DTtoM); +} -static npy_int64 asfreq_WtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_WtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_WtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_WtoDT, asfreq_DTtoW); +} static npy_int64 asfreq_WtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; + int tempStore = af_info->targetFreq; + af_info->targetFreq = FR_DAY; if (dInfoCalc_SetFromAbsDate(&dinfo, - asfreq_WtoD(ordinal, relation, af_info) + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + asfreq_WtoDT(ordinal, relation, af_info) + ORD_OFFSET, + GREGORIAN_CALENDAR)) return INT_ERR_CODE; + af_info->targetFreq = tempStore; if (relation == 'S') { - return DtoB_WeekendToMonday(dinfo.absdate, dinfo.day_of_week); - } + return DtoB_WeekendToMonday(dinfo.absdate, dinfo.day_of_week); + } else { - return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); - } + return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); + } } -static npy_int64 asfreq_WtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoH(asfreq_WtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_WtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoT(asfreq_WtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_WtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoS(asfreq_WtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } - //************ FROM MONTHLY *************** static void MtoD_ym(npy_int64 ordinal, int *y, int *m) { *y = floordiv(ordinal, 12) + BASE_YEAR; @@ -555,49 +624,50 @@ static void MtoD_ym(npy_int64 ordinal, int *y, int *m) { } -static npy_int64 asfreq_MtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) { - +static npy_int64 asfreq_MtoDT(npy_int64 ordinal, char relation, asfreq_info* af_info) { npy_int64 absdate; int y, m; - if (relation == 'S') { - MtoD_ym(ordinal, &y, &m); - if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; - return absdate - ORD_OFFSET; - } else { - MtoD_ym(ordinal + 1, &y, &m); - if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; - return absdate - 1 - ORD_OFFSET; + if (relation == 'E') { + ordinal += 1; + } + MtoD_ym(ordinal, &y, &m); + if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; + ordinal = absdate - ORD_OFFSET; + + if (relation == 'E') { + ordinal -= 1; } + + return convert_daytime(ordinal, FR_DAY, af_info->targetFreq, relation != 'S'); } static npy_int64 asfreq_MtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoA(asfreq_MtoD(ordinal, 'E', &NULL_AF_INFO), relation, af_info); } + return transform_via_day(ordinal, relation, af_info, asfreq_MtoDT, asfreq_DTtoA); +} static npy_int64 asfreq_MtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoQ(asfreq_MtoD(ordinal, 'E', &NULL_AF_INFO), relation, af_info); } + return transform_via_day(ordinal, relation, af_info, asfreq_MtoDT, asfreq_DTtoQ); +} -static npy_int64 asfreq_MtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_MtoD(ordinal, relation, &NULL_AF_INFO), relation, af_info); } +static npy_int64 asfreq_MtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_MtoDT, asfreq_DTtoW); +} static npy_int64 asfreq_MtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { - struct date_info dinfo; + + int tempStore = af_info->targetFreq; + af_info->targetFreq = FR_DAY; if (dInfoCalc_SetFromAbsDate(&dinfo, - asfreq_MtoD(ordinal, relation, &NULL_AF_INFO) + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + asfreq_MtoDT(ordinal, relation, af_info) + ORD_OFFSET, + GREGORIAN_CALENDAR)) return INT_ERR_CODE; + af_info->targetFreq = tempStore; if (relation == 'S') { return DtoB_WeekendToMonday(dinfo.absdate, dinfo.day_of_week); } else { return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); } } -static npy_int64 asfreq_MtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoH(asfreq_MtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_MtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoT(asfreq_MtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_MtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoS(asfreq_MtoD(ordinal, relation, &NULL_AF_INFO), relation, &NULL_AF_INFO); } - //************ FROM QUARTERLY *************** static void QtoD_ym(npy_int64 ordinal, int *y, int *m, asfreq_info *af_info) { @@ -611,122 +681,124 @@ static void QtoD_ym(npy_int64 ordinal, int *y, int *m, asfreq_info *af_info) { } } -static npy_int64 asfreq_QtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) { +static npy_int64 asfreq_QtoDT(npy_int64 ordinal, char relation, asfreq_info *af_info) { npy_int64 absdate; int y, m; - if (relation == 'S') { - QtoD_ym(ordinal, &y, &m, af_info); - // printf("ordinal: %d, year: %d, month: %d\n", (int) ordinal, y, m); - if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; - return absdate - ORD_OFFSET; - } else { - QtoD_ym(ordinal+1, &y, &m, af_info); - /* printf("ordinal: %d, year: %d, month: %d\n", (int) ordinal, y, m); */ - if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; - return absdate - 1 - ORD_OFFSET; + if (relation == 'E') { + ordinal += 1; } + + QtoD_ym(ordinal, &y, &m, af_info); + + if ((absdate = absdate_from_ymd(y, m, 1)) == INT_ERR_CODE) return INT_ERR_CODE; + + if (relation == 'E') { + absdate -= 1; + } + + return convert_daytime(absdate - ORD_OFFSET, FR_DAY, af_info->targetFreq, relation != 'S'); } -static npy_int64 asfreq_QtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_QtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_QtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_QtoDT, asfreq_DTtoQ); +} static npy_int64 asfreq_QtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoA(asfreq_QtoD(ordinal, relation, af_info), relation, af_info); } + return transform_via_day(ordinal, relation, af_info, asfreq_QtoDT, asfreq_DTtoA); +} static npy_int64 asfreq_QtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { - return asfreq_DtoM(asfreq_QtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } + return transform_via_day(ordinal, relation, af_info, asfreq_QtoDT, asfreq_DTtoM); +} -static npy_int64 asfreq_QtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_QtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_QtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_QtoDT, asfreq_DTtoW); +} static npy_int64 asfreq_QtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; + int tempStore = af_info->targetFreq; + af_info->targetFreq = FR_DAY; if (dInfoCalc_SetFromAbsDate(&dinfo, - asfreq_QtoD(ordinal, relation, af_info) + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + asfreq_QtoDT(ordinal, relation, af_info) + ORD_OFFSET, + GREGORIAN_CALENDAR)) return INT_ERR_CODE; + af_info->targetFreq = tempStore; if (relation == 'S') { return DtoB_WeekendToMonday(dinfo.absdate, dinfo.day_of_week); } else { return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); } } -static npy_int64 asfreq_QtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoH(asfreq_QtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_QtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoT(asfreq_QtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_QtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoS(asfreq_QtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } - - //************ FROM ANNUAL *************** -static npy_int64 asfreq_AtoD(npy_int64 ordinal, char relation, asfreq_info *af_info) { - npy_int64 absdate, final_adj; - int year; +static npy_int64 asfreq_AtoDT(npy_int64 year, char relation, asfreq_info *af_info) { + npy_int64 absdate; int month = (af_info->from_a_year_end) % 12; - // start from 1970 - ordinal += BASE_YEAR; + // start from 1970 + year += BASE_YEAR; - if (month == 0) { month = 1; } - else { month += 1; } + month += 1; - if (relation == 'S') { - if (af_info->from_a_year_end == 12) {year = ordinal;} - else {year = ordinal - 1;} - final_adj = 0; - } else { - if (af_info->from_a_year_end == 12) {year = ordinal+1;} - else {year = ordinal;} - final_adj = -1; + if (af_info->from_a_year_end != 12) { + year -= 1; + } + + if (relation == 'E') { + year += 1; } + absdate = absdate_from_ymd(year, month, 1); + if (absdate == INT_ERR_CODE) { - return INT_ERR_CODE; - } - return absdate + final_adj - ORD_OFFSET; + return INT_ERR_CODE; + } + + if (relation == 'E') { + absdate -= 1; + } + + return convert_daytime(absdate - ORD_OFFSET, FR_DAY, af_info->targetFreq, relation != 'S'); } -static npy_int64 asfreq_AtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoA(asfreq_AtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_AtoA(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_AtoDT, asfreq_DTtoA); +} -static npy_int64 asfreq_AtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoQ(asfreq_AtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_AtoQ(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_AtoDT, asfreq_DTtoQ); +} -static npy_int64 asfreq_AtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoM(asfreq_AtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_AtoM(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_AtoDT, asfreq_DTtoM); +} -static npy_int64 asfreq_AtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoW(asfreq_AtoD(ordinal, relation, af_info), relation, af_info); } +static npy_int64 asfreq_AtoW(npy_int64 ordinal, char relation, asfreq_info *af_info) { + return transform_via_day(ordinal, relation, af_info, asfreq_AtoDT, asfreq_DTtoW); +} static npy_int64 asfreq_AtoB(npy_int64 ordinal, char relation, asfreq_info *af_info) { struct date_info dinfo; + int tempStore = af_info->targetFreq; + af_info->targetFreq = FR_DAY; if (dInfoCalc_SetFromAbsDate(&dinfo, - asfreq_AtoD(ordinal, relation, af_info) + ORD_OFFSET, - GREGORIAN_CALENDAR)) return INT_ERR_CODE; + asfreq_AtoDT(ordinal, relation, af_info) + ORD_OFFSET, + GREGORIAN_CALENDAR)) return INT_ERR_CODE; + af_info->targetFreq = tempStore; if (relation == 'S') { return DtoB_WeekendToMonday(dinfo.absdate, dinfo.day_of_week); } else { return DtoB_WeekendToFriday(dinfo.absdate, dinfo.day_of_week); } } -static npy_int64 asfreq_AtoH(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoH(asfreq_AtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_AtoT(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoT(asfreq_AtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } -static npy_int64 asfreq_AtoS(npy_int64 ordinal, char relation, asfreq_info *af_info) - { return asfreq_DtoS(asfreq_AtoD(ordinal, relation, af_info), relation, &NULL_AF_INFO); } - static npy_int64 nofunc(npy_int64 ordinal, char relation, asfreq_info *af_info) { return INT_ERR_CODE; } static npy_int64 no_op(npy_int64 ordinal, char relation, asfreq_info *af_info) { return ordinal; } // end of frequency specific conversion routines -static int get_freq_group(int freq) { return (freq/1000)*1000; } - static int calc_a_year_end(int freq, int group) { int result = (freq - group) % 12; if (result == 0) {return 12;} @@ -741,30 +813,33 @@ void get_asfreq_info(int fromFreq, int toFreq, asfreq_info *af_info) { int fromGroup = get_freq_group(fromFreq); int toGroup = get_freq_group(toFreq); + af_info->sourceFreq = fromFreq; + af_info->targetFreq = toFreq; + switch(fromGroup) { - case FR_WK: { + case FR_WK: af_info->from_week_end = calc_week_end(fromFreq, fromGroup); - } break; - case FR_ANN: { + break; + case FR_ANN: af_info->from_a_year_end = calc_a_year_end(fromFreq, fromGroup); - } break; - case FR_QTR: { + break; + case FR_QTR: af_info->from_q_year_end = calc_a_year_end(fromFreq, fromGroup); - } break; + break; } switch(toGroup) { - case FR_WK: { + case FR_WK: af_info->to_week_end = calc_week_end(toFreq, toGroup); - } break; - case FR_ANN: { + break; + case FR_ANN: af_info->to_a_year_end = calc_a_year_end(toFreq, toGroup); - } break; - case FR_QTR: { + break; + case FR_QTR: af_info->to_q_year_end = calc_a_year_end(toFreq, toGroup); - } break; + break; } } @@ -786,10 +861,15 @@ freq_conv_func get_asfreq_func(int fromFreq, int toFreq) case FR_MTH: return &asfreq_AtoM; case FR_WK: return &asfreq_AtoW; case FR_BUS: return &asfreq_AtoB; - case FR_DAY: return &asfreq_AtoD; - case FR_HR: return &asfreq_AtoH; - case FR_MIN: return &asfreq_AtoT; - case FR_SEC: return &asfreq_AtoS; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_AtoDT; + default: return &nofunc; } @@ -801,10 +881,14 @@ freq_conv_func get_asfreq_func(int fromFreq, int toFreq) case FR_MTH: return &asfreq_QtoM; case FR_WK: return &asfreq_QtoW; case FR_BUS: return &asfreq_QtoB; - case FR_DAY: return &asfreq_QtoD; - case FR_HR: return &asfreq_QtoH; - case FR_MIN: return &asfreq_QtoT; - case FR_SEC: return &asfreq_QtoS; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_QtoDT; default: return &nofunc; } @@ -816,10 +900,14 @@ freq_conv_func get_asfreq_func(int fromFreq, int toFreq) case FR_MTH: return &no_op; case FR_WK: return &asfreq_MtoW; case FR_BUS: return &asfreq_MtoB; - case FR_DAY: return &asfreq_MtoD; - case FR_HR: return &asfreq_MtoH; - case FR_MIN: return &asfreq_MtoT; - case FR_SEC: return &asfreq_MtoS; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_MtoDT; default: return &nofunc; } @@ -831,10 +919,14 @@ freq_conv_func get_asfreq_func(int fromFreq, int toFreq) case FR_MTH: return &asfreq_WtoM; case FR_WK: return &asfreq_WtoW; case FR_BUS: return &asfreq_WtoB; - case FR_DAY: return &asfreq_WtoD; - case FR_HR: return &asfreq_WtoH; - case FR_MIN: return &asfreq_WtoT; - case FR_SEC: return &asfreq_WtoS; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_WtoDT; default: return &nofunc; } @@ -845,112 +937,84 @@ freq_conv_func get_asfreq_func(int fromFreq, int toFreq) case FR_QTR: return &asfreq_BtoQ; case FR_MTH: return &asfreq_BtoM; case FR_WK: return &asfreq_BtoW; - case FR_DAY: return &asfreq_BtoD; case FR_BUS: return &no_op; - case FR_HR: return &asfreq_BtoH; - case FR_MIN: return &asfreq_BtoT; - case FR_SEC: return &asfreq_BtoS; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_BtoDT; default: return &nofunc; } case FR_DAY: - switch(toGroup) - { - case FR_ANN: return &asfreq_DtoA; - case FR_QTR: return &asfreq_DtoQ; - case FR_MTH: return &asfreq_DtoM; - case FR_WK: return &asfreq_DtoW; - case FR_BUS: return &asfreq_DtoB; - case FR_DAY: return &asfreq_DtoD; - case FR_HR: return &asfreq_DtoH; - case FR_MIN: return &asfreq_DtoT; - case FR_SEC: return &asfreq_DtoS; - default: return &nofunc; - } - case FR_HR: - switch(toGroup) - { - case FR_ANN: return &asfreq_HtoA; - case FR_QTR: return &asfreq_HtoQ; - case FR_MTH: return &asfreq_HtoM; - case FR_WK: return &asfreq_HtoW; - case FR_BUS: return &asfreq_HtoB; - case FR_DAY: return &asfreq_HtoD; - case FR_HR: return &no_op; - case FR_MIN: return &asfreq_HtoT; - case FR_SEC: return &asfreq_HtoS; - default: return &nofunc; - } - case FR_MIN: - switch(toGroup) - { - case FR_ANN: return &asfreq_TtoA; - case FR_QTR: return &asfreq_TtoQ; - case FR_MTH: return &asfreq_TtoM; - case FR_WK: return &asfreq_TtoW; - case FR_BUS: return &asfreq_TtoB; - case FR_DAY: return &asfreq_TtoD; - case FR_HR: return &asfreq_TtoH; - case FR_MIN: return &no_op; - case FR_SEC: return &asfreq_TtoS; - default: return &nofunc; - } - case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: switch(toGroup) { - case FR_ANN: return &asfreq_StoA; - case FR_QTR: return &asfreq_StoQ; - case FR_MTH: return &asfreq_StoM; - case FR_WK: return &asfreq_StoW; - case FR_BUS: return &asfreq_StoB; - case FR_DAY: return &asfreq_StoD; - case FR_HR: return &asfreq_StoH; - case FR_MIN: return &asfreq_StoT; - case FR_SEC: return &no_op; + case FR_ANN: return &asfreq_DTtoA; + case FR_QTR: return &asfreq_DTtoQ; + case FR_MTH: return &asfreq_DTtoM; + case FR_WK: return &asfreq_DTtoW; + case FR_BUS: return &asfreq_DTtoB; + case FR_DAY: + case FR_HR: + case FR_MIN: + case FR_SEC: + case FR_MS: + case FR_US: + case FR_NS: + return &asfreq_WithinDT; default: return &nofunc; } + default: return &nofunc; } } -double get_abs_time(int freq, npy_int64 daily_ord, npy_int64 ordinal) { +double get_abs_time(int freq, npy_int64 date_ordinal, npy_int64 ordinal) { + //printf("get_abs_time %d %lld %lld\n", freq, date_ordinal, ordinal); - npy_int64 start_ord, per_day, unit; - switch(freq) - { - case FR_HR: - per_day = 24; - unit = 60 * 60; - break; - case FR_MIN: - per_day = 24*60; - unit = 60; - break; - case FR_SEC: - per_day = 24*60*60; - unit = 1; - break; - default: - return 0; // 24*60*60 - 1; + int freq_index, day_index, base_index; + npy_int64 per_day, start_ord; + double unit, result; + + if (freq <= FR_DAY) { + return 0; } - start_ord = asfreq_DtoHIGHFREQ(daily_ord, 'S', per_day); - /* printf("start_ord: %d\n", start_ord); */ - return (double) ( unit * (ordinal - start_ord)); - /* if (ordinal >= 0) { */ - /* } */ - /* else { */ - /* return (double) (unit * mod_compat(ordinal - start_ord, per_day)); */ - /* } */ + freq_index = get_freq_group_index(freq); + day_index = get_freq_group_index(FR_DAY); + base_index = get_freq_group_index(FR_SEC); + + //printf(" indices: day %d, freq %d, base %d\n", day_index, freq_index, base_index); + + per_day = get_daytime_conversion_factor(day_index, freq_index); + unit = get_daytime_conversion_factor(freq_index, base_index); + + //printf(" per_day: %lld, unit: %f\n", per_day, unit); + + if (base_index < freq_index) { + unit = 1 / unit; + //printf(" corrected unit: %f\n", unit); + } + + start_ord = date_ordinal * per_day; + //printf("start_ord: %lld\n", start_ord); + result = (double) ( unit * (ordinal - start_ord)); + //printf(" result: %f\n", result); + return result; } /* Sets the time part of the DateTime object. */ -static -int dInfoCalc_SetFromAbsTime(struct date_info *dinfo, - double abstime) +static int dInfoCalc_SetFromAbsTime(struct date_info *dinfo, + double abstime) { int inttime; int hour,minute; @@ -973,18 +1037,16 @@ int dInfoCalc_SetFromAbsTime(struct date_info *dinfo, /* Set the instance's value using the given date and time. calendar may be set to the flags: GREGORIAN_CALENDAR, JULIAN_CALENDAR to indicate the calendar to be used. */ -static -int dInfoCalc_SetFromAbsDateTime(struct date_info *dinfo, - npy_int64 absdate, - double abstime, - int calendar) +static int dInfoCalc_SetFromAbsDateTime(struct date_info *dinfo, + npy_int64 absdate, + double abstime, + int calendar) { - /* Bounds check */ Py_AssertWithArg(abstime >= 0.0 && abstime <= SECONDS_PER_DAY, - PyExc_ValueError, - "abstime out of range (0.0 - 86400.0): %f", - abstime); + PyExc_ValueError, + "abstime out of range (0.0 - 86400.0): %f", + abstime); /* Calculate the date */ if (dInfoCalc_SetFromAbsDate(dinfo, absdate, calendar)) goto onError; @@ -993,7 +1055,7 @@ int dInfoCalc_SetFromAbsDateTime(struct date_info *dinfo, if (dInfoCalc_SetFromAbsTime(dinfo, abstime)) goto onError; return 0; - onError: +onError: return INT_ERR_CODE; } @@ -1007,15 +1069,16 @@ npy_int64 asfreq(npy_int64 period_ordinal, int freq1, int freq2, char relation) freq_conv_func func; asfreq_info finfo; - func = get_asfreq_func(freq1, freq2); + func = get_asfreq_func(freq1, freq2); + get_asfreq_info(freq1, freq2, &finfo); val = (*func)(period_ordinal, relation, &finfo); if (val == INT_ERR_CODE) { - // Py_Error(PyExc_ValueError, "Unable to convert to desired frequency."); - goto onError; - } + //Py_Error(PyExc_ValueError, "Unable to convert to desired frequency."); + goto onError; + } return val; onError: return INT_ERR_CODE; @@ -1024,19 +1087,33 @@ npy_int64 asfreq(npy_int64 period_ordinal, int freq1, int freq2, char relation) /* generate an ordinal in period space */ npy_int64 get_period_ordinal(int year, int month, int day, - int hour, int minute, int second, - int freq) + int hour, int minute, int second, int microseconds, int picoseconds, + int freq) { - npy_int64 absdays, delta; + npy_int64 absdays, delta, seconds; npy_int64 weeks, days; npy_int64 ordinal, day_adj; int freq_group, fmonth, mdiff; freq_group = get_freq_group(freq); - if (freq == FR_SEC) { + if (freq == FR_SEC || freq == FR_MS || freq == FR_US || freq == FR_NS) { + absdays = absdate_from_ymd(year, month, day); delta = (absdays - ORD_OFFSET); - return (npy_int64)(delta*86400 + hour*3600 + minute*60 + second); + seconds = (npy_int64)(delta * 86400 + hour * 3600 + minute * 60 + second); + + switch(freq) { + case FR_MS: + return seconds * 1000 + microseconds / 1000; + + case FR_US: + return seconds * 1000000 + microseconds; + + case FR_NS: + return seconds * 1000000000 + microseconds * 1000 + picoseconds / 1000; + } + + return seconds; } if (freq == FR_MIN) { @@ -1056,12 +1133,12 @@ npy_int64 get_period_ordinal(int year, int month, int day, if (freq == FR_DAY) { - return (npy_int64) (absdate_from_ymd(year, month, day) - ORD_OFFSET); + return (npy_int64) (absdate_from_ymd(year, month, day) - ORD_OFFSET); } if (freq == FR_UND) { - return (npy_int64) (absdate_from_ymd(year, month, day) - ORD_OFFSET); + return (npy_int64) (absdate_from_ymd(year, month, day) - ORD_OFFSET); } if (freq == FR_BUS) @@ -1091,26 +1168,26 @@ npy_int64 get_period_ordinal(int year, int month, int day, if (freq_group == FR_QTR) { - fmonth = freq - FR_QTR; - if (fmonth == 0) fmonth = 12; + fmonth = freq - FR_QTR; + if (fmonth == 0) fmonth = 12; - mdiff = month - fmonth; - if (mdiff < 0) mdiff += 12; - if (month >= fmonth) mdiff += 12; + mdiff = month - fmonth; + if (mdiff < 0) mdiff += 12; + if (month >= fmonth) mdiff += 12; - return (year - BASE_YEAR) * 4 + (mdiff - 1) / 3; + return (year - BASE_YEAR) * 4 + (mdiff - 1) / 3; } if (freq_group == FR_ANN) { - fmonth = freq - FR_ANN; - if (fmonth == 0) fmonth = 12; - if (month <= fmonth) { - return year - BASE_YEAR; - } - else { - return year - BASE_YEAR + 1; - } + fmonth = freq - FR_ANN; + if (fmonth == 0) fmonth = 12; + if (month <= fmonth) { + return year - BASE_YEAR; + } + else { + return year - BASE_YEAR + 1; + } } Py_Error(PyExc_RuntimeError, "Unable to generate frequency ordinal"); @@ -1120,22 +1197,23 @@ npy_int64 get_period_ordinal(int year, int month, int day, } /* - Returns the proleptic Gregorian ordinal of the date, as an integer. - This corresponds to the number of days since Jan., 1st, 1AD. - When the instance has a frequency less than daily, the proleptic date - is calculated for the last day of the period. -*/ + Returns the proleptic Gregorian ordinal of the date, as an integer. + This corresponds to the number of days since Jan., 1st, 1AD. + When the instance has a frequency less than daily, the proleptic date + is calculated for the last day of the period. + */ npy_int64 get_python_ordinal(npy_int64 period_ordinal, int freq) { asfreq_info af_info; - npy_int64 (*toDaily)(npy_int64, char, asfreq_info*); + freq_conv_func toDaily = NULL; if (freq == FR_DAY) return period_ordinal + ORD_OFFSET; toDaily = get_asfreq_func(freq, FR_DAY); get_asfreq_info(freq, FR_DAY, &af_info); + return toDaily(period_ordinal, 'E', &af_info) + ORD_OFFSET; } @@ -1147,8 +1225,8 @@ char *str_replace(const char *s, const char *old, const char *new) { for (i = 0; s[i] != '\0'; i++) { if (strstr(&s[i], old) == &s[i]) { - count++; - i += oldlen - 1; + count++; + i += oldlen - 1; } } @@ -1256,12 +1334,12 @@ static int _ISOWeek(struct date_info *dinfo) if (week < 0) { /* The day lies in last week of the previous year */ if ((week > -2) || - (week == -2 && dInfoCalc_Leapyear(dinfo->year-1, dinfo->calendar))) + (week == -2 && dInfoCalc_Leapyear(dinfo->year-1, dinfo->calendar))) week = 53; else week = 52; } else if (week == 53) { - /* Check if the week belongs to year or year+1 */ + /* Check if the week belongs to year or year+1 */ if (31-dinfo->day + dinfo->day_of_week < 3) { week = 1; } @@ -1273,15 +1351,19 @@ static int _ISOWeek(struct date_info *dinfo) int get_date_info(npy_int64 ordinal, int freq, struct date_info *dinfo) { npy_int64 absdate = get_python_ordinal(ordinal, freq); - /* printf("freq: %d, absdate: %d\n", freq, (int) absdate); */ double abstime = get_abs_time(freq, absdate - ORD_OFFSET, ordinal); - if (abstime < 0) { - abstime += 86400; - absdate -= 1; - } + + while (abstime < 0) { + abstime += 86400; + absdate -= 1; + } + while (abstime >= 86400) { + abstime -= 86400; + absdate += 1; + } if(dInfoCalc_SetFromAbsDateTime(dinfo, absdate, - abstime, GREGORIAN_CALENDAR)) + abstime, GREGORIAN_CALENDAR)) return INT_ERR_CODE; return 0; diff --git a/pandas/src/period.h b/pandas/src/period.h index 4ba92bf8fde41..af35838ad0355 100644 --- a/pandas/src/period.h +++ b/pandas/src/period.h @@ -85,6 +85,9 @@ #define FR_HR 7000 /* Hourly */ #define FR_MIN 8000 /* Minutely */ #define FR_SEC 9000 /* Secondly */ +#define FR_MS 10000 /* Millisecondly */ +#define FR_US 11000 /* Microsecondly */ +#define FR_NS 12000 /* Nanosecondly */ #define FR_UND -10000 /* Undefined */ @@ -102,6 +105,9 @@ typedef struct asfreq_info { int from_q_year_end; // month the year ends on in the "from" frequency int to_q_year_end; // month the year ends on in the "to" frequency + + int sourceFreq; + int targetFreq; } asfreq_info; @@ -130,7 +136,7 @@ typedef npy_int64 (*freq_conv_func)(npy_int64, char, asfreq_info*); npy_int64 asfreq(npy_int64 period_ordinal, int freq1, int freq2, char relation); npy_int64 get_period_ordinal(int year, int month, int day, - int hour, int minute, int second, + int hour, int minute, int second, int microseconds, int picoseconds, int freq); npy_int64 get_python_ordinal(npy_int64 period_ordinal, int freq); diff --git a/pandas/tests/test_format.py b/pandas/tests/test_format.py index 9405f3c58bfd7..55f70e9e4fe28 100644 --- a/pandas/tests/test_format.py +++ b/pandas/tests/test_format.py @@ -1664,7 +1664,7 @@ def test_timedelta64(self): from pandas import date_range from datetime import datetime, timedelta - Series(np.array([1100, 20], dtype='timedelta64[s]')).to_string() + Series(np.array([1100, 20], dtype='timedelta64[ns]')).to_string() s = Series(date_range('2012-1-1', periods=3, freq='D')) diff --git a/pandas/tests/test_index.py b/pandas/tests/test_index.py index 852d02764affc..f29cee6942672 100644 --- a/pandas/tests/test_index.py +++ b/pandas/tests/test_index.py @@ -29,6 +29,8 @@ import pandas as pd from pandas.lib import Timestamp +from pandas import _np_version_under1p7 + class TestIndex(unittest.TestCase): _multiprocess_can_split_ = True @@ -230,6 +232,25 @@ def test_asof(self): d = self.dateIndex[0].to_datetime() tm.assert_isinstance(self.dateIndex.asof(d), Timestamp) + def test_nanosecond_index_access(self): + if _np_version_under1p7: + import nose + + raise nose.SkipTest('numpy >= 1.7 required') + + from pandas import Series, Timestamp, DatetimeIndex + + s = Series([Timestamp('20130101')]).values.view('i8')[0] + r = DatetimeIndex([s + 50 + i for i in range(100)]) + x = Series(np.random.randn(100), index=r) + + first_value = x.asof(x.index[0]) + + # this does not yet work, as parsing strings is done via dateutil + #self.assertEqual(first_value, x['2013-01-01 00:00:00.000000050+0000']) + + self.assertEqual(first_value, x[Timestamp(np.datetime64('2013-01-01 00:00:00.000000050+0000', 'ns'))]) + def test_argsort(self): result = self.strIndex.argsort() expected = np.array(self.strIndex).argsort() diff --git a/pandas/tseries/converter.py b/pandas/tseries/converter.py index 7c34562e64f6e..bfbd28f7bb4a4 100644 --- a/pandas/tseries/converter.py +++ b/pandas/tseries/converter.py @@ -458,7 +458,13 @@ def _daily_finder(vmin, vmax, freq): periodsperday = -1 if freq >= FreqGroup.FR_HR: - if freq == FreqGroup.FR_SEC: + if freq == FreqGroup.FR_NS: + periodsperday = 24 * 60 * 60 * 1000000000 + elif freq == FreqGroup.FR_US: + periodsperday = 24 * 60 * 60 * 1000000 + elif freq == FreqGroup.FR_MS: + periodsperday = 24 * 60 * 60 * 1000 + elif freq == FreqGroup.FR_SEC: periodsperday = 24 * 60 * 60 elif freq == FreqGroup.FR_MIN: periodsperday = 24 * 60 diff --git a/pandas/tseries/frequencies.py b/pandas/tseries/frequencies.py index 2c4fc0d1b9c78..d1fd51c073f83 100644 --- a/pandas/tseries/frequencies.py +++ b/pandas/tseries/frequencies.py @@ -12,6 +12,7 @@ import pandas.core.common as com import pandas.lib as lib import pandas.tslib as tslib +from pandas import _np_version_under1p7 class FreqGroup(object): @@ -24,6 +25,9 @@ class FreqGroup(object): FR_HR = 7000 FR_MIN = 8000 FR_SEC = 9000 + FR_MS = 10000 + FR_US = 11000 + FR_NS = 12000 class Resolution(object): @@ -116,7 +120,7 @@ def _get_freq_str(base, mult=1): # Offset names ("time rules") and related functions -from pandas.tseries.offsets import (Micro, Milli, Second, Minute, Hour, +from pandas.tseries.offsets import (Nano, Micro, Milli, Second, Minute, Hour, Day, BDay, CDay, Week, MonthBegin, MonthEnd, BMonthBegin, BMonthEnd, QuarterBegin, QuarterEnd, BQuarterBegin, @@ -275,6 +279,9 @@ def _get_freq_str(base, mult=1): } +if not _np_version_under1p7: + _offset_map['N'] = Nano() + _offset_to_period_map = { 'WEEKDAY': 'D', 'EOM': 'M', @@ -291,6 +298,9 @@ def _get_freq_str(base, mult=1): 'B': 'B', 'T': 'T', 'S': 'S', + 'L': 'L', + 'U': 'U', + 'N': 'N', 'H': 'H', 'Q': 'Q', 'A': 'A', @@ -609,6 +619,9 @@ def get_standard_freq(freq): "H": 7000, # Hourly "T": 8000, # Minutely "S": 9000, # Secondly + "L": 10000, # Millisecondly + "U": 11000, # Microsecondly + "N": 12000, # Nanosecondly } _reverse_period_code_map = {} @@ -636,7 +649,10 @@ def _period_alias_dictionary(): H_aliases = ["H", "HR", "HOUR", "HRLY", "HOURLY"] T_aliases = ["T", "MIN", "MINUTE", "MINUTELY"] S_aliases = ["S", "SEC", "SECOND", "SECONDLY"] - + L_aliases = ["L", "MS", "MILLISECOND", "MILLISECONDLY"] + U_aliases = ["U", "US", "MICROSECOND", "MICROSECONDLY"] + N_aliases = ["N", "NS", "NANOSECOND", "NANOSECONDLY"] + for k in M_aliases: alias_dict[k] = 'M' @@ -655,6 +671,15 @@ def _period_alias_dictionary(): for k in S_aliases: alias_dict[k] = 'S' + for k in L_aliases: + alias_dict[k] = 'L' + + for k in U_aliases: + alias_dict[k] = 'U' + + for k in N_aliases: + alias_dict[k] = 'N' + A_prefixes = ["A", "Y", "ANN", "ANNUAL", "ANNUALLY", "YR", "YEAR", "YEARLY"] @@ -722,6 +747,9 @@ def _period_alias_dictionary(): "hour": "H", "minute": "T", "second": "S", + "millisecond": "L", + "microsecond": "U", + "nanosecond": "N", } diff --git a/pandas/tseries/index.py b/pandas/tseries/index.py index 847896871045b..24e94f4c2d482 100644 --- a/pandas/tseries/index.py +++ b/pandas/tseries/index.py @@ -1162,8 +1162,16 @@ def get_value(self, series, key): Fast lookup of value from 1-dimensional ndarray. Only use this if you know what you're doing """ + timestamp = None + #if isinstance(key, Timestamp): + # timestamp = key + #el if isinstance(key, datetime): - return self.get_value_maybe_box(series, key) + # needed to localize naive datetimes + timestamp = Timestamp(key, tz=self.tz) + + if timestamp: + return self.get_value_maybe_box(series, timestamp) try: return _maybe_box(self, Index.get_value(self, series, key), series, key) diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 232ebd2c3726c..e496bf46cf57a 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -8,6 +8,8 @@ # import after tools, dateutil check from dateutil.relativedelta import relativedelta import pandas.tslib as tslib +import numpy as np +from pandas import _np_version_under1p7 __all__ = ['Day', 'BusinessDay', 'BDay', 'CustomBusinessDay', 'CDay', 'MonthBegin', 'BMonthBegin', 'MonthEnd', 'BMonthEnd', @@ -25,7 +27,6 @@ class ApplyTypeError(TypeError): class CacheableOffset(object): - _cacheable = True @@ -107,7 +108,7 @@ def _should_cache(self): def _params(self): attrs = [(k, v) for k, v in compat.iteritems(vars(self)) if k not in ['kwds', '_offset', 'name', 'normalize', - 'busdaycalendar']] + 'busdaycalendar']] attrs.extend(list(self.kwds.items())) attrs = sorted(set(attrs)) @@ -123,7 +124,7 @@ def __repr__(self): attrs = [] for attr in sorted(self.__dict__): if ((attr == 'kwds' and len(self.kwds) == 0) - or attr.startswith('_')): + or attr.startswith('_')): continue elif attr == 'kwds': kwds_new = {} @@ -157,6 +158,7 @@ def __eq__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset + other = to_offset(other) if not isinstance(other, DateOffset): @@ -255,6 +257,7 @@ class BusinessDay(CacheableOffset, DateOffset): """ DateOffset subclass representing possibly n business days """ + def __init__(self, n=1, **kwds): self.n = int(n) self.kwds = kwds @@ -412,6 +415,7 @@ class CustomBusinessDay(BusinessDay): def __init__(self, n=1, **kwds): # Check we have the required numpy version from distutils.version import LooseVersion + if LooseVersion(np.__version__) < '1.7.0': raise NotImplementedError("CustomBusinessDay requires numpy >= " "1.7.0. Current version: " + @@ -476,7 +480,7 @@ def apply(self, other): day64 = dt64.astype('datetime64[D]') time = dt64 - day64 - if self.n<=0: + if self.n <= 0: roll = 'forward' else: roll = 'backward' @@ -558,7 +562,7 @@ def apply(self, other): wkday, days_in_month = tslib.monthrange(other.year, other.month) lastBDay = days_in_month - max(((wkday + days_in_month - 1) - % 7) - 4, 0) + % 7) - 4, 0) if n > 0 and not other.day >= lastBDay: n = n - 1 @@ -621,6 +625,7 @@ class Week(CacheableOffset, DateOffset): weekday : int, default None Always generate specific day of week. 0 for Monday """ + def __init__(self, n=1, **kwds): self.n = n self.weekday = kwds.get('weekday', None) @@ -628,7 +633,7 @@ def __init__(self, n=1, **kwds): if self.weekday is not None: if self.weekday < 0 or self.weekday > 6: raise ValueError('Day must be 0<=day<=6, got %d' % - self.weekday) + self.weekday) self._inc = timedelta(weeks=1) self.kwds = kwds @@ -667,6 +672,7 @@ def rule_code(self): suffix = '-%s' % (_weekday_dict[self.weekday]) return 'W' + suffix + _weekday_dict = { 0: 'MON', 1: 'TUE', @@ -696,6 +702,7 @@ class WeekOfMonth(CacheableOffset, DateOffset): 5: Saturdays 6: Sundays """ + def __init__(self, n=1, **kwds): self.n = n self.weekday = kwds['weekday'] @@ -706,10 +713,10 @@ def __init__(self, n=1, **kwds): if self.weekday < 0 or self.weekday > 6: raise ValueError('Day must be 0<=day<=6, got %d' % - self.weekday) + self.weekday) if self.week < 0 or self.week > 3: raise ValueError('Week must be 0<=day<=3, got %d' % - self.week) + self.week) self.kwds = kwds @@ -773,7 +780,7 @@ def apply(self, other): wkday, days_in_month = tslib.monthrange(other.year, other.month) lastBDay = days_in_month - max(((wkday + days_in_month - 1) - % 7) - 4, 0) + % 7) - 4, 0) monthsToGo = 3 - ((other.month - self.startingMonth) % 3) if monthsToGo == 3: @@ -1068,7 +1075,7 @@ def _decrement(date): def _rollf(date): if (date.month != self.month or - date.day < tslib.monthrange(date.year, date.month)[1]): + date.day < tslib.monthrange(date.year, date.month)[1]): date = _increment(date) return date @@ -1120,7 +1127,7 @@ def _increment(date): def _decrement(date): year = date.year if date.month < self.month or (date.month == self.month and - date.day == 1): + date.day == 1): year -= 1 return datetime(year, self.month, 1, date.hour, date.minute, date.second, date.microsecond) @@ -1164,6 +1171,7 @@ def rule_code(self): def _tick_comp(op): def f(self, other): return op(self.delta, other.delta) + return f @@ -1191,6 +1199,7 @@ def __add__(self, other): def __eq__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset + other = to_offset(other) if isinstance(other, Tick): @@ -1206,6 +1215,7 @@ def __hash__(self): def __ne__(self, other): if isinstance(other, compat.string_types): from pandas.tseries.frequencies import to_offset + other = to_offset(other) if isinstance(other, Tick): @@ -1265,8 +1275,11 @@ def _delta_to_tick(delta): def _delta_to_nanoseconds(delta): - if isinstance(delta, Tick): + if isinstance(delta, np.timedelta64): + return delta.astype('timedelta64[ns]').item() + elif isinstance(delta, Tick): delta = delta.delta + return (delta.days * 24 * 60 * 60 * 1000000 + delta.seconds * 1000000 + delta.microseconds) * 1000 @@ -1302,9 +1315,10 @@ class Micro(Tick): class Nano(Tick): - _inc = 1 + _inc = np.timedelta64(1, 'ns') if not _np_version_under1p7 else 1 _rule_base = 'N' + BDay = BusinessDay BMonthEnd = BusinessMonthEnd BMonthBegin = BusinessMonthBegin @@ -1355,6 +1369,7 @@ def generate_range(start=None, end=None, periods=None, """ if time_rule is not None: from pandas.tseries.frequencies import get_offset + offset = get_offset(time_rule) start = to_datetime(start) diff --git a/pandas/tseries/period.py b/pandas/tseries/period.py index b6f3c3c83f3d8..cd81867ff8f08 100644 --- a/pandas/tseries/period.py +++ b/pandas/tseries/period.py @@ -125,7 +125,7 @@ def __init__(self, value=None, freq=None, ordinal=None, if self.ordinal is None: self.ordinal = tslib.period_ordinal(dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, + dt.hour, dt.minute, dt.second, dt.microsecond, 0, base) self.freq = _freq_mod._get_freq_str(base) @@ -447,6 +447,11 @@ def _get_date_and_freq(value, freq): freq = 'T' elif reso == 'second': freq = 'S' + elif reso == 'microsecond': + if dt.microsecond % 1000 == 0: + freq = 'L' + else: + freq = 'U' else: raise ValueError("Invalid frequency or could not infer: %s" % reso) @@ -1238,7 +1243,7 @@ def _range_from_fields(year=None, month=None, quarter=None, day=None, year, quarter = _make_field_arrays(year, quarter) for y, q in zip(year, quarter): y, m = _quarter_to_myear(y, q, freq) - val = tslib.period_ordinal(y, m, 1, 1, 1, 1, base) + val = tslib.period_ordinal(y, m, 1, 1, 1, 1, 0, 0, base) ordinals.append(val) else: base, mult = _gfc(freq) @@ -1247,7 +1252,7 @@ def _range_from_fields(year=None, month=None, quarter=None, day=None, arrays = _make_field_arrays(year, month, day, hour, minute, second) for y, mth, d, h, mn, s in zip(*arrays): - ordinals.append(tslib.period_ordinal(y, mth, d, h, mn, s, base)) + ordinals.append(tslib.period_ordinal(y, mth, d, h, mn, s, 0, 0, base)) return np.array(ordinals, dtype=np.int64), freq @@ -1276,7 +1281,7 @@ def _ordinal_from_fields(year, month, quarter, day, hour, minute, if quarter is not None: year, month = _quarter_to_myear(year, quarter, freq) - return tslib.period_ordinal(year, month, day, hour, minute, second, base) + return tslib.period_ordinal(year, month, day, hour, minute, second, 0, 0, base) def _quarter_to_myear(year, quarter, freq): diff --git a/pandas/tseries/tests/test_frequencies.py b/pandas/tseries/tests/test_frequencies.py index 6386f61a24a85..00a3d392a45c0 100644 --- a/pandas/tseries/tests/test_frequencies.py +++ b/pandas/tseries/tests/test_frequencies.py @@ -8,7 +8,7 @@ import numpy as np -from pandas import Index, DatetimeIndex, date_range, period_range +from pandas import Index, DatetimeIndex, Timestamp, date_range, period_range from pandas.tseries.frequencies import to_offset, infer_freq from pandas.tseries.tools import to_datetime @@ -17,6 +17,8 @@ import pandas.lib as lib +from pandas import _np_version_under1p7 + def test_to_offset_multiple(): freqstr = '2h30min' @@ -47,6 +49,12 @@ def test_to_offset_multiple(): expected = offsets.Milli(10075) assert(result == expected) + if not _np_version_under1p7: + freqstr = '2800N' + result = to_offset(freqstr) + expected = offsets.Nano(2800) + assert(result == expected) + # malformed try: to_offset('2h20m') @@ -116,13 +124,12 @@ def test_microsecond(self): self._check_tick(timedelta(microseconds=1), 'U') def test_nanosecond(self): - idx = DatetimeIndex(np.arange(0, 100, 10)) - inferred = idx.inferred_freq - - self.assert_(inferred == '10N') + if _np_version_under1p7: + raise nose.SkipTest("requires numpy >= 1.7 to run") + self._check_tick(np.timedelta64(1, 'ns'), 'N') def _check_tick(self, base_delta, code): - b = datetime.now() + b = Timestamp(datetime.now()) for i in range(1, 5): inc = base_delta * i index = _dti([b + inc * j for j in range(3)]) diff --git a/pandas/tseries/tests/test_offsets.py b/pandas/tseries/tests/test_offsets.py index 5b4e3251683bb..a77b0afb20b52 100644 --- a/pandas/tseries/tests/test_offsets.py +++ b/pandas/tseries/tests/test_offsets.py @@ -27,6 +27,8 @@ import pandas.util.testing as tm from pandas.tseries.offsets import BusinessMonthEnd, CacheableOffset +from pandas import _np_version_under1p7 + _multiprocess_can_split_ = True @@ -1620,14 +1622,15 @@ def test_onOffset(self): def assertEq(offset, base, expected): actual = offset + base + actual_swapped = base + offset try: assert actual == expected + assert actual_swapped == expected except AssertionError: raise AssertionError("\nExpected: %s\nActual: %s\nFor Offset: %s)" "\nAt Date: %s" % (expected, actual, offset, base)) - def test_Hour(): assertEq(Hour(), datetime(2010, 1, 1), datetime(2010, 1, 1, 1)) assertEq(Hour(-1), datetime(2010, 1, 1, 1), datetime(2010, 1, 1)) @@ -1668,6 +1671,58 @@ def test_Second(): assert not Second().isAnchored() +def test_Millisecond(): + assertEq(Milli(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 1000)) + assertEq(Milli(-1), datetime(2010, 1, 1, 0, 0, 0, 1000), datetime(2010, 1, 1)) + assertEq(Milli(2), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 2000)) + assertEq(2 * Milli(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 2000)) + assertEq(-1 * Milli(), datetime(2010, 1, 1, 0, 0, 0, 1000), datetime(2010, 1, 1)) + + assert (Milli(3) + Milli(2)) == Milli(5) + assert (Milli(3) - Milli(2)) == Milli() + + +def test_MillisecondTimestampArithmetic(): + assertEq(Milli(), Timestamp('2010-01-01'), Timestamp('2010-01-01 00:00:00.001')) + assertEq(Milli(-1), Timestamp('2010-01-01 00:00:00.001'), Timestamp('2010-01-01')) + + +def test_Microsecond(): + assertEq(Micro(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 1)) + assertEq(Micro(-1), datetime(2010, 1, 1, 0, 0, 0, 1), datetime(2010, 1, 1)) + assertEq(2 * Micro(), datetime(2010, 1, 1), datetime(2010, 1, 1, 0, 0, 0, 2)) + assertEq(-1 * Micro(), datetime(2010, 1, 1, 0, 0, 0, 1), datetime(2010, 1, 1)) + + assert (Micro(3) + Micro(2)) == Micro(5) + assert (Micro(3) - Micro(2)) == Micro() + + +def test_NanosecondGeneric(): + timestamp = Timestamp(datetime(2010, 1, 1)) + assert timestamp.nanosecond == 0 + + result = timestamp + Nano(10) + assert result.nanosecond == 10 + + reverse_result = Nano(10) + timestamp + assert reverse_result.nanosecond == 10 + + +def test_Nanosecond(): + if _np_version_under1p7: + import nose + raise nose.SkipTest('numpy >= 1.7 required') + + timestamp = Timestamp(datetime(2010, 1, 1)) + assertEq(Nano(), timestamp, timestamp + np.timedelta64(1, 'ns')) + assertEq(Nano(-1), timestamp + np.timedelta64(1, 'ns'), timestamp) + assertEq(2 * Nano(), timestamp, timestamp + np.timedelta64(2, 'ns')) + assertEq(-1 * Nano(), timestamp + np.timedelta64(1, 'ns'), timestamp) + + assert (Nano(3) + Nano(2)) == Nano(5) + assert (Nano(3) - Nano(2)) == Nano() + + def test_tick_offset(): assert not Day().isAnchored() assert not Milli().isAnchored() diff --git a/pandas/tseries/tests/test_period.py b/pandas/tseries/tests/test_period.py index 173ebeb199b3b..9abecc0aeeec6 100644 --- a/pandas/tseries/tests/test_period.py +++ b/pandas/tseries/tests/test_period.py @@ -195,7 +195,7 @@ def test_period_constructor(self): self.assertRaises(ValueError, Period, ordinal=200701) - self.assertRaises(KeyError, Period, '2007-1-1', freq='U') + self.assertRaises(KeyError, Period, '2007-1-1', freq='X') def test_freq_str(self): i1 = Period('1982', freq='Min') @@ -208,6 +208,16 @@ def test_repr(self): p = Period('2000-12-15') self.assert_('2000-12-15' in repr(p)) + def test_millisecond_repr(self): + p = Period('2000-01-01 12:15:02.123') + + self.assertEquals("Period('2000-01-01 12:15:02.123', 'L')", repr(p)) + + def test_microsecond_repr(self): + p = Period('2000-01-01 12:15:02.123567') + + self.assertEquals("Period('2000-01-01 12:15:02.123567', 'U')", repr(p)) + def test_strftime(self): p = Period('2000-1-1 12:34:12', freq='S') res = p.strftime('%Y-%m-%d %H:%M:%S') @@ -466,7 +476,14 @@ def test_constructor_infer_freq(self): p = Period('2007-01-01 07:10:15') self.assert_(p.freq == 'S') - self.assertRaises(ValueError, Period, '2007-01-01 07:10:15.123456') + p = Period('2007-01-01 07:10:15.123') + self.assert_(p.freq == 'L') + + p = Period('2007-01-01 07:10:15.123000') + self.assert_(p.freq == 'L') + + p = Period('2007-01-01 07:10:15.123400') + self.assert_(p.freq == 'U') def noWrap(item): @@ -1115,9 +1132,9 @@ def test_constructor_field_arrays(self): self.assert_(idx.equals(exp)) def test_constructor_U(self): - # U was used as undefined period + # X was used as undefined period self.assertRaises(KeyError, period_range, '2007-1-1', periods=500, - freq='U') + freq='X') def test_constructor_arrays_negative_year(self): years = np.arange(1960, 2000).repeat(4) @@ -2168,6 +2185,15 @@ def test_minutely(self): def test_secondly(self): self._check_freq('S', '1970-01-01') + + def test_millisecondly(self): + self._check_freq('L', '1970-01-01') + + def test_microsecondly(self): + self._check_freq('U', '1970-01-01') + + def test_nanosecondly(self): + self._check_freq('N', '1970-01-01') def _check_freq(self, freq, base_date): rng = PeriodIndex(start=base_date, periods=10, freq=freq) diff --git a/pandas/tseries/tests/test_timeseries.py b/pandas/tseries/tests/test_timeseries.py index 07903f9a9374a..e4504420bacc2 100644 --- a/pandas/tseries/tests/test_timeseries.py +++ b/pandas/tseries/tests/test_timeseries.py @@ -13,7 +13,7 @@ randn = np.random.randn from pandas import (Index, Series, TimeSeries, DataFrame, - isnull, date_range, Timestamp, DatetimeIndex, + isnull, date_range, Timestamp, Period, DatetimeIndex, Int64Index, to_datetime, bdate_range) from pandas.core.daterange import DateRange @@ -43,6 +43,7 @@ from pandas.core.datetools import BDay import pandas.core.common as com from pandas import concat +from pandas import _np_version_under1p7 from numpy.testing.decorators import slow @@ -663,7 +664,7 @@ def test_index_cast_datetime64_other_units(self): def test_index_astype_datetime64(self): idx = Index([datetime(2012, 1, 1)], dtype=object) - if np.__version__ >= LooseVersion('1.7'): + if not _np_version_under1p7: raise nose.SkipTest("test only valid in numpy < 1.7") casted = idx.astype(np.dtype('M8[D]')) @@ -1332,6 +1333,28 @@ def test_to_period(self): pts = ts.to_period('M') self.assert_(pts.index.equals(exp.index.asfreq('M'))) + def create_dt64_based_index(self): + data = [Timestamp('2007-01-01 10:11:12.123456Z'), + Timestamp('2007-01-01 10:11:13.789123Z')] + index = DatetimeIndex(data) + return index + + def test_to_period_millisecond(self): + index = self.create_dt64_based_index() + + period = index.to_period(freq='L') + self.assertEqual(2, len(period)) + self.assertEqual(period[0], Period('2007-01-01 10:11:12.123Z', 'L')) + self.assertEqual(period[1], Period('2007-01-01 10:11:13.789Z', 'L')) + + def test_to_period_microsecond(self): + index = self.create_dt64_based_index() + + period = index.to_period(freq='U') + self.assertEqual(2, len(period)) + self.assertEqual(period[0], Period('2007-01-01 10:11:12.123456Z', 'U')) + self.assertEqual(period[1], Period('2007-01-01 10:11:13.789123Z', 'U')) + def test_to_period_tz(self): _skip_if_no_pytz() from dateutil.tz import tzlocal @@ -1598,7 +1621,7 @@ def test_frame_datetime64_handling_groupby(self): (3, np.datetime64('2012-07-04'))], columns=['a', 'date']) result = df.groupby('a').first() - self.assertEqual(result['date'][3], datetime(2012,7,3)) + self.assertEqual(result['date'][3], Timestamp('2012-07-03')) def test_series_interpolate_intraday(self): # #1698 @@ -2020,6 +2043,27 @@ def test_join_self(self): joined = index.join(index, how=kind) self.assert_(index is joined) + def assert_index_parameters(self, index): + assert index.freq == '40960N' + assert index.inferred_freq == '40960N' + + def test_ns_index(self): + + if _np_version_under1p7: + raise nose.SkipTest + + nsamples = 400 + ns = int(1e9 / 24414) + dtstart = np.datetime64('2012-09-20T00:00:00') + + dt = dtstart + np.arange(nsamples) * np.timedelta64(ns, 'ns') + freq = ns * pd.datetools.Nano() + index = pd.DatetimeIndex(dt, freq=freq, name='time') + self.assert_index_parameters(index) + + new_index = pd.DatetimeIndex(start=index[0], end=index[-1], freq=index.freq) + self.assert_index_parameters(new_index) + class TestDatetime64(unittest.TestCase): """ diff --git a/pandas/tseries/tests/test_timeseries_legacy.py b/pandas/tseries/tests/test_timeseries_legacy.py index d1f4f647db0e1..2e8418d8f50b2 100644 --- a/pandas/tseries/tests/test_timeseries_legacy.py +++ b/pandas/tseries/tests/test_timeseries_legacy.py @@ -255,6 +255,7 @@ def test_rule_aliases(self): rule = datetools.to_offset('10us') self.assert_(rule == datetools.Micro(10)) + class TestLegacyCompat(unittest.TestCase): def setUp(self): diff --git a/pandas/tseries/tests/test_tslib.py b/pandas/tseries/tests/test_tslib.py index b9a7356412a10..4e7daede03085 100644 --- a/pandas/tseries/tests/test_tslib.py +++ b/pandas/tseries/tests/test_tslib.py @@ -1,10 +1,20 @@ import unittest +import nose import numpy as np from pandas import tslib from datetime import datetime +from pandas.core.api import Timestamp + +from pandas.tslib import period_asfreq + +from pandas.tseries.frequencies import get_freq + +from pandas import _np_version_under1p7 + + class TestDatetimeParsingWrappers(unittest.TestCase): def test_verify_datetime_bounds(self): for year in (1, 1000, 1677, 2262, 5000): @@ -46,6 +56,7 @@ def test_does_not_convert_mixed_integer(self): tslib._does_string_look_like_datetime(good_date_string) ) + class TestArrayToDatetime(unittest.TestCase): def test_parsing_valid_dates(self): arr = np.array(['01-01-2013', '01-02-2013'], dtype=object) @@ -118,6 +129,67 @@ def test_coerce_of_invalid_datetimes(self): ) ) + +class TestTimestamp(unittest.TestCase): + def setUp(self): + if _np_version_under1p7: + raise nose.SkipTest('numpy >= 1.7 required') + self.timestamp = Timestamp(datetime.utcnow()) + + def assert_ns_timedelta(self, modified_timestamp, expected_value): + value = self.timestamp.value + modified_value = modified_timestamp.value + + self.assertEquals(modified_value - value, expected_value) + + def test_timedelta_ns_arithmetic(self): + self.assert_ns_timedelta(self.timestamp + np.timedelta64(-123, 'ns'), -123) + + def test_timedelta_ns_based_arithmetic(self): + self.assert_ns_timedelta(self.timestamp + np.timedelta64(1234567898, 'ns'), 1234567898) + + def test_timedelta_us_arithmetic(self): + self.assert_ns_timedelta(self.timestamp + np.timedelta64(-123, 'us'), -123000) + + def test_timedelta_ns_arithmetic(self): + time = self.timestamp + np.timedelta64(-123, 'ms') + self.assert_ns_timedelta(time, -123000000) + + def test_nanosecond_string_parsing(self): + self.timestamp = Timestamp('2013-05-01 07:15:45.123456789') + self.assertEqual(self.timestamp.value, 1367392545123456000) + + +class TestTslib(unittest.TestCase): + + def test_intraday_conversion_factors(self): + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('H'), False), 24) + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('T'), False), 1440) + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('S'), False), 86400) + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('L'), False), 86400000) + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('U'), False), 86400000000) + self.assertEqual(period_asfreq(1, get_freq('D'), get_freq('N'), False), 86400000000000) + + self.assertEqual(period_asfreq(1, get_freq('H'), get_freq('T'), False), 60) + self.assertEqual(period_asfreq(1, get_freq('H'), get_freq('S'), False), 3600) + self.assertEqual(period_asfreq(1, get_freq('H'), get_freq('L'), False), 3600000) + self.assertEqual(period_asfreq(1, get_freq('H'), get_freq('U'), False), 3600000000) + self.assertEqual(period_asfreq(1, get_freq('H'), get_freq('N'), False), 3600000000000) + + self.assertEqual(period_asfreq(1, get_freq('T'), get_freq('S'), False), 60) + self.assertEqual(period_asfreq(1, get_freq('T'), get_freq('L'), False), 60000) + self.assertEqual(period_asfreq(1, get_freq('T'), get_freq('U'), False), 60000000) + self.assertEqual(period_asfreq(1, get_freq('T'), get_freq('N'), False), 60000000000) + + self.assertEqual(period_asfreq(1, get_freq('S'), get_freq('L'), False), 1000) + self.assertEqual(period_asfreq(1, get_freq('S'), get_freq('U'), False), 1000000) + self.assertEqual(period_asfreq(1, get_freq('S'), get_freq('N'), False), 1000000000) + + self.assertEqual(period_asfreq(1, get_freq('L'), get_freq('U'), False), 1000) + self.assertEqual(period_asfreq(1, get_freq('L'), get_freq('N'), False), 1000000) + + self.assertEqual(period_asfreq(1, get_freq('U'), get_freq('N'), False), 1000) + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], exit=False) diff --git a/pandas/tslib.pyx b/pandas/tslib.pyx index 99b09446be232..0df0fc377d000 100644 --- a/pandas/tslib.pyx +++ b/pandas/tslib.pyx @@ -19,10 +19,12 @@ cdef extern from "Python.h": cdef PyTypeObject *Py_TYPE(object) int PySlice_Check(object) +# this is our datetime.pxd +from datetime cimport * +from util cimport is_integer_object, is_datetime64_object, is_timedelta64_object from libc.stdlib cimport free -from util cimport is_integer_object, is_datetime64_object cimport util from datetime cimport * @@ -332,6 +334,9 @@ class NaTType(_NaT): def __repr__(self): return 'NaT' + def __hash__(self): + return iNaT + def weekday(self): return -1 @@ -568,23 +573,27 @@ cdef class _Timestamp(datetime): dts.us, ts.tzinfo) def __add__(self, other): + if is_timedelta64_object(other): + return Timestamp(self.value + other.astype('timedelta64[ns]').item(), tz=self.tzinfo) + if is_integer_object(other): if self.offset is None: + return Timestamp(self.value + other, tz=self.tzinfo) msg = ("Cannot add integral value to Timestamp " "without offset.") raise ValueError(msg) else: return Timestamp((self.offset.__mul__(other)).apply(self)) - else: - if isinstance(other, timedelta) or hasattr(other, 'delta'): - nanos = _delta_to_nanoseconds(other) - return Timestamp(self.value + nanos, tz=self.tzinfo) - else: - result = datetime.__add__(self, other) - if isinstance(result, datetime): - result = Timestamp(result) - result.nanosecond = self.nanosecond - return result + + if isinstance(other, timedelta) or hasattr(other, 'delta'): + nanos = _delta_to_nanoseconds(other) + return Timestamp(self.value + nanos, tz=self.tzinfo) + + result = datetime.__add__(self, other) + if isinstance(result, datetime): + result = Timestamp(result) + result.nanosecond = self.nanosecond + return result def __sub__(self, other): if is_integer_object(other): @@ -636,10 +645,12 @@ cdef class _NaT(_Timestamp): def _delta_to_nanoseconds(delta): - try: + if hasattr(delta, 'delta'): delta = delta.delta - except: - pass + if is_timedelta64_object(delta): + return delta.astype("timedelta64[ns]").item() + if is_integer_object(delta): + return delta return (delta.days * 24 * 60 * 60 * 1000000 + delta.seconds * 1000000 + delta.microseconds) * 1000 @@ -2140,7 +2151,7 @@ cdef ndarray[int64_t] localize_dt64arr_to_period(ndarray[int64_t] stamps, continue pandas_datetime_to_datetimestruct(stamps[i], PANDAS_FR_ns, &dts) result[i] = get_period_ordinal(dts.year, dts.month, dts.day, - dts.hour, dts.min, dts.sec, freq) + dts.hour, dts.min, dts.sec, dts.us, dts.ps, freq) elif _is_tzlocal(tz): for i in range(n): @@ -2155,7 +2166,7 @@ cdef ndarray[int64_t] localize_dt64arr_to_period(ndarray[int64_t] stamps, pandas_datetime_to_datetimestruct(stamps[i] + delta, PANDAS_FR_ns, &dts) result[i] = get_period_ordinal(dts.year, dts.month, dts.day, - dts.hour, dts.min, dts.sec, freq) + dts.hour, dts.min, dts.sec, dts.us, dts.ps, freq) else: # Adjust datetime64 timestamp, recompute datetimestruct trans = _get_transitions(tz) @@ -2174,7 +2185,7 @@ cdef ndarray[int64_t] localize_dt64arr_to_period(ndarray[int64_t] stamps, pandas_datetime_to_datetimestruct(stamps[i] + deltas[0], PANDAS_FR_ns, &dts) result[i] = get_period_ordinal(dts.year, dts.month, dts.day, - dts.hour, dts.min, dts.sec, freq) + dts.hour, dts.min, dts.sec, dts.us, dts.ps, freq) else: for i in range(n): if stamps[i] == NPY_NAT: @@ -2183,7 +2194,7 @@ cdef ndarray[int64_t] localize_dt64arr_to_period(ndarray[int64_t] stamps, pandas_datetime_to_datetimestruct(stamps[i] + deltas[pos[i]], PANDAS_FR_ns, &dts) result[i] = get_period_ordinal(dts.year, dts.month, dts.day, - dts.hour, dts.min, dts.sec, freq) + dts.hour, dts.min, dts.sec, dts.us, dts.ps, freq) return result @@ -2220,7 +2231,7 @@ cdef extern from "period.h": void get_asfreq_info(int fromFreq, int toFreq, asfreq_info *af_info) int64_t get_period_ordinal(int year, int month, int day, - int hour, int minute, int second, + int hour, int minute, int second, int microseconds, int picoseconds, int freq) except INT32_MIN int64_t get_python_ordinal(int64_t period_ordinal, int freq) except INT32_MIN @@ -2284,7 +2295,7 @@ def dt64arr_to_periodarr(ndarray[int64_t] dtarr, int freq, tz=None): for i in range(l): pandas_datetime_to_datetimestruct(dtarr[i], PANDAS_FR_ns, &dts) out[i] = get_period_ordinal(dts.year, dts.month, dts.day, - dts.hour, dts.min, dts.sec, freq) + dts.hour, dts.min, dts.sec, dts.us, dts.ps, freq) else: out = localize_dt64arr_to_period(dtarr, freq, tz) return out @@ -2318,7 +2329,7 @@ cpdef int64_t period_asfreq(int64_t period_ordinal, int freq1, int freq2, """ cdef: int64_t retval - + if end: retval = asfreq(period_ordinal, freq1, freq2, END) else: @@ -2361,17 +2372,18 @@ def period_asfreq_arr(ndarray[int64_t] arr, int freq1, int freq2, bint end): return result -def period_ordinal(int y, int m, int d, int h, int min, int s, int freq): +def period_ordinal(int y, int m, int d, int h, int min, int s, int us, int ps, int freq): cdef: int64_t ordinal - return get_period_ordinal(y, m, d, h, min, s, freq) + return get_period_ordinal(y, m, d, h, min, s, us, ps, freq) cpdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq): cdef: pandas_datetimestruct dts date_info dinfo + float subsecond_fraction get_date_info(ordinal, freq, &dinfo) @@ -2381,7 +2393,9 @@ cpdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq): dts.hour = dinfo.hour dts.min = dinfo.minute dts.sec = int(dinfo.second) - dts.us = dts.ps = 0 + subsecond_fraction = dinfo.second - dts.sec + dts.us = int((subsecond_fraction) * 1e6) + dts.ps = int(((subsecond_fraction) * 1e6 - dts.us) * 1e6) return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts) @@ -2411,6 +2425,12 @@ def period_format(int64_t value, int freq, object fmt=None): fmt = b'%Y-%m-%d %H:%M' elif freq_group == 9000: # SEC fmt = b'%Y-%m-%d %H:%M:%S' + elif freq_group == 10000: # MILLISEC + fmt = b'%Y-%m-%d %H:%M:%S.%l' + elif freq_group == 11000: # MICROSEC + fmt = b'%Y-%m-%d %H:%M:%S.%u' + elif freq_group == 12000: # NANOSEC + fmt = b'%Y-%m-%d %H:%M:%S.%n' else: raise ValueError('Unknown freq: %d' % freq) @@ -2419,9 +2439,12 @@ def period_format(int64_t value, int freq, object fmt=None): cdef list extra_fmts = [(b"%q", b"^`AB`^"), (b"%f", b"^`CD`^"), - (b"%F", b"^`EF`^")] + (b"%F", b"^`EF`^"), + (b"%l", b"^`GH`^"), + (b"%u", b"^`IJ`^"), + (b"%n", b"^`KL`^")] -cdef list str_extra_fmts = ["^`AB`^", "^`CD`^", "^`EF`^"] +cdef list str_extra_fmts = ["^`AB`^", "^`CD`^", "^`EF`^", "^`GH`^", "^`IJ`^", "^`KL`^"] cdef _period_strftime(int64_t value, int freq, object fmt): import sys @@ -2460,6 +2483,12 @@ cdef _period_strftime(int64_t value, int freq, object fmt): repl = '%.2d' % (year % 100) elif i == 2: repl = '%d' % year + elif i == 3: + repl = '%03d' % (value % 1000) + elif i == 4: + repl = '%06d' % (value % 1000000) + elif i == 5: + repl = '%09d' % (value % 1000000000) result = result.replace(str_extra_fmts[i], repl) diff --git a/test.sh b/test.sh index 324ac68d66b73..4a9ffd7be98b1 100755 --- a/test.sh +++ b/test.sh @@ -1,5 +1,6 @@ #!/bin/sh -coverage erase +command -v coverage >/dev/null && coverage erase +command -v python-coverage >/dev/null && python-coverage erase # nosetests pandas/tests/test_index.py --with-coverage --cover-package=pandas.core --pdb-failure --pdb #nosetests -w pandas --with-coverage --cover-package=pandas --pdb-failure --pdb #--cover-inclusive #nosetests -A "not slow" -w pandas/tseries --with-coverage --cover-package=pandas.tseries $* #--cover-inclusive