From 618f3c2c52bb3d257e123a5c329eea42f29aaa36 Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Sun, 21 Aug 2016 19:50:15 +0200 Subject: [PATCH 01/15] lmdb: Sync to upstream. --- src/lmdb/lmdb.h | 36 ++++- src/lmdb/mdb.c | 351 +++++++++++++++++++++++------------------------- src/lmdb/midl.h | 9 +- 3 files changed, 197 insertions(+), 199 deletions(-) diff --git a/src/lmdb/lmdb.h b/src/lmdb/lmdb.h index 85dc50d580c..30d586203ac 100644 --- a/src/lmdb/lmdb.h +++ b/src/lmdb/lmdb.h @@ -167,6 +167,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -179,11 +180,30 @@ typedef int mdb_mode_t; typedef mode_t mdb_mode_t; #endif -#ifdef MDB_VL32 -typedef uint64_t mdb_size_t; -#define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ +#ifdef _WIN32 +# define MDB_FMT_Z "I" #else +# define MDB_FMT_Z "z" /**< printf/scanf format modifier for size_t */ +#endif + +#ifndef MDB_VL32 +/** Unsigned type used for mapsize, entry counts and page/transaction IDs. + * + * It is normally size_t, hence the name. Defining MDB_VL32 makes it + * uint64_t, but do not try this unless you know what you are doing. + */ typedef size_t mdb_size_t; +# define MDB_SIZE_MAX SIZE_MAX /**< max #mdb_size_t */ +/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */ +# define MDB_PRIy(t) MDB_FMT_Z #t +/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */ +# define MDB_SCNy(t) MDB_FMT_Z #t +#else +typedef uint64_t mdb_size_t; +# define MDB_SIZE_MAX UINT64_MAX +# define MDB_PRIy(t) PRI##t##64 +# define MDB_SCNy(t) SCN##t##64 +# define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ #endif /** An abstraction for a file handle. @@ -320,7 +340,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_REVERSEKEY 0x02 /** use sorted duplicates */ #define MDB_DUPSORT 0x04 - /** numeric keys in native byte order: either unsigned int or size_t. + /** numeric keys in native byte order, either unsigned int or #mdb_size_t. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) * The keys must all be of the same size. */ #define MDB_INTEGERKEY 0x08 /** with #MDB_DUPSORT, sorted dup items have fixed size */ @@ -456,8 +477,10 @@ typedef enum MDB_cursor_op { #define MDB_BAD_VALSIZE (-30781) /** The specified DBI was changed unexpectedly */ #define MDB_BAD_DBI (-30780) + /** Unexpected problem - txn should abort */ +#define MDB_PROBLEM (-30779) /** The last defined error code */ -#define MDB_LAST_ERRCODE MDB_BAD_DBI +#define MDB_LAST_ERRCODE MDB_PROBLEM /** @} */ /** @brief Statistics for a database in the environment */ @@ -1105,7 +1128,8 @@ int mdb_txn_renew(MDB_txn *txn); * keys must be unique and may have only a single data item. *
  • #MDB_INTEGERKEY * Keys are binary integers in native byte order, either unsigned int - * or size_t, and will be sorted as such. + * or #mdb_size_t, and will be sorted as such. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) * The keys must all be of the same size. *
  • #MDB_DUPFIXED * This flag may only be used in combination with #MDB_DUPSORT. This option diff --git a/src/lmdb/mdb.c b/src/lmdb/mdb.c index c6d95a37991..6b00917ac02 100644 --- a/src/lmdb/mdb.c +++ b/src/lmdb/mdb.c @@ -232,7 +232,7 @@ union semun { #if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) # error "Unknown or unsupported endianness (BYTE_ORDER)" -#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#elif (-6 & 5) || CHAR_BIT!=8 || UINT_MAX!=0xffffffff || MDB_SIZE_MAX%UINT_MAX # error "Two's complement, reasonably sized integer types, please" #endif @@ -359,12 +359,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t; #else #define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 #endif -#define Z "I" #else #define THREAD_RET void * #define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) #define THREAD_FINISH(thr) pthread_join(thr,NULL) -#define Z "z" /**< printf format modifier for size_t */ /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ #define MDB_PIDLOCK 1 @@ -462,15 +460,9 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; #define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif +#define Z MDB_FMT_Z /**< printf/scanf format modifier for size_t */ +#define Yu MDB_PRIy(u) /**< printf format for #mdb_size_t */ +#define Yd MDB_PRIy(d) /**< printf format for "signed #mdb_size_t" */ #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) #define MNAME_LEN 32 @@ -880,9 +872,23 @@ typedef struct MDB_txninfo { + (((MDB_PIDLOCK) != 0) << 16))) /** @} */ -/** Common header for all page types. - * Overflow records occupy a number of contiguous pages with no - * headers on any page after the first. +/** Common header for all page types. The page type depends on #mp_flags. + * + * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with + * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages + * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. + * + * #P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of #F_BIGDATA nodes. + * + * #P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. + * + * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a freeDB record. */ typedef struct MDB_page { #define mp_pgno mp_p.p_pgno @@ -891,7 +897,7 @@ typedef struct MDB_page { pgno_t p_pgno; /**< page number */ struct MDB_page *p_next; /**< for in-memory list of freed pages */ } mp_p; - uint16_t mp_pad; + uint16_t mp_pad; /**< key size if this is a LEAF2 page */ /** @defgroup mdb_page Page Flags * @ingroup internal * Flags for the page headers. @@ -958,7 +964,9 @@ typedef struct MDB_page { /** The number of overflow pages needed to store the given size. */ #define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - /** Link in #MDB_txn.%mt_loose_pgs list */ + /** Link in #MDB_txn.%mt_loose_pgs list. + * Kept outside the page header, which is needed when reusing the page. + */ #define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) /** Header for a single key/data pair within a page. @@ -1041,7 +1049,7 @@ typedef struct MDB_node { #ifdef MISALIGNED_OK #define COPY_PGNO(dst,src) dst = src #else -#if SIZE_MAX > 4294967295UL +#if MDB_SIZE_MAX > 0xffffffffU #define COPY_PGNO(dst,src) do { \ unsigned short *s, *d; \ s = (unsigned short *)&(src); \ @@ -1086,9 +1094,9 @@ typedef struct MDB_db { pgno_t md_root; /**< the root page of this tree */ } MDB_db; - /** mdb_dbi_open flags */ #define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ #define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) + /** #mdb_dbi_open() flags */ #define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) @@ -1127,7 +1135,10 @@ typedef struct MDB_meta { #define mm_psize mm_dbs[FREE_DBI].md_pad /** Any persistent environment flags. @ref mdb_env */ #define mm_flags mm_dbs[FREE_DBI].md_flags - pgno_t mm_last_pg; /**< last used page in file */ + /** Last used page in the datafile. + * Actually the file may be shorter if the freeDB lists the final pages. + */ + pgno_t mm_last_pg; volatile txnid_t mm_txnid; /**< txnid that committed this page */ } MDB_meta; @@ -1180,7 +1191,7 @@ struct MDB_txn { * in this transaction, linked through #NEXT_LOOSE_PAGE(page). */ MDB_page *mt_loose_pgs; - /* #Number of loose pages (#mt_loose_pgs) */ + /** Number of loose pages (#mt_loose_pgs) */ int mt_loose_count; /** The sorted list of dirty pages we temporarily wrote to disk * because the dirty list was full. page numbers in here are @@ -1317,6 +1328,11 @@ struct MDB_cursor { indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ #ifdef MDB_VL32 MDB_page *mc_ovpg; /**< a referenced overflow page */ +# define MC_OVPG(mc) ((mc)->mc_ovpg) +# define MC_SET_OVPG(mc, pg) ((mc)->mc_ovpg = (pg)) +#else +# define MC_OVPG(mc) ((MDB_page *)0) +# define MC_SET_OVPG(mc, pg) ((void)0) #endif }; @@ -1522,13 +1538,18 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; /** @endcond */ -/** Compare two items pointing at size_t's of unknown alignment. */ +/** Compare two items pointing at '#mdb_size_t's of unknown alignment. */ #ifdef MISALIGNED_OK # define mdb_cmp_clong mdb_cmp_long #else # define mdb_cmp_clong mdb_cmp_cint #endif +/** True if we need #mdb_cmp_clong() instead of \b cmp for #MDB_INTEGERDUP */ +#define NEED_CMP_CLONG(cmp, ksize) \ + (UINT_MAX < MDB_SIZE_MAX && \ + (cmp) == mdb_cmp_int && (ksize) == sizeof(mdb_size_t)) + #ifdef _WIN32 static SECURITY_DESCRIPTOR mdb_null_sd; static SECURITY_ATTRIBUTES mdb_all_sa; @@ -1569,6 +1590,7 @@ static char *const mdb_errstr[] = { "MDB_BAD_TXN: Transaction must abort, has a child, or is invalid", "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", + "MDB_PROBLEM: Unexpected problem - txn should abort", }; char * @@ -1713,20 +1735,20 @@ mdb_page_list(MDB_page *mp) case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Y"u pages %u%s\n", + fprintf(stderr, "Overflow page %"Yu" pages %u%s\n", pgno, mp->mp_pages, state); return; case P_META: - fprintf(stderr, "Meta-page %"Y"u txnid %"Y"u\n", + fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n", pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); return; default: - fprintf(stderr, "Bad page %"Y"u flags 0x%u\n", pgno, mp->mp_flags); + fprintf(stderr, "Bad page %"Yu" flags 0x%u\n", pgno, mp->mp_flags); return; } nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Y"u numkeys %d%s\n", type, pgno, nkeys, state); + fprintf(stderr, "%s %"Yu" numkeys %d%s\n", type, pgno, nkeys, state); for (i=0; imn_data; nsize = NODESIZE + key.mv_size; if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Y"u, %s\n", i, NODEPGNO(node), + fprintf(stderr, "key %d: page %"Yu", %s\n", i, NODEPGNO(node), DKEY(&key)); total += nsize; } else { @@ -1837,7 +1859,7 @@ static void mdb_audit(MDB_txn *txn) } } if (freecount + count + NUM_METAS != txn->mt_next_pgno) { - fprintf(stderr, "audit: %"Y"u freecount: %"Y"u count: %"Y"u total: %"Y"u next_pgno: %"Y"u\n", + fprintf(stderr, "audit: %"Yu" freecount: %"Yu" count: %"Yu" total: %"Yu" next_pgno: %"Yu"\n", txn->mt_txnid, freecount, count+NUM_METAS, freecount+count+NUM_METAS, txn->mt_next_pgno); } @@ -1854,10 +1876,8 @@ int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) { MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && a->mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, a->mv_size)) dcmp = mdb_cmp_clong; -#endif return dcmp(a, b); } @@ -1972,8 +1992,14 @@ mdb_cursor_unref(MDB_cursor *mc) mc->mc_pg[0] = NULL; mc->mc_flags &= ~C_INITIALIZED; } +#define MDB_CURSOR_UNREF(mc, force) \ + (((force) || ((mc)->mc_flags & C_INITIALIZED)) \ + ? mdb_cursor_unref(mc) \ + : (void)0) + #else #define MDB_PAGE_UNREF(txn, mp) +#define MDB_CURSOR_UNREF(mc, force) ((void)0) #endif /* MDB_VL32 */ /** Loosen or free a single page. @@ -2005,7 +2031,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) if (mp != dl[x].mptr) { /* bad cursor? */ mc->mc_flags &= ~(C_INITIALIZED|C_EOF); txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } /* ok, it's ours */ loose = 1; @@ -2017,8 +2043,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) } } if (loose) { - DPRINTF(("loosen db %d page %"Y"u", DDBI(mc), - mp->mp_pgno)); + DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno)); NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; txn->mt_loose_pgs = mp; txn->mt_loose_count++; @@ -2315,8 +2340,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) np = txn->mt_loose_pgs; txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Y"u", DDBI(mc), - np->mp_pgno)); + DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno)); *mp = np; return MDB_SUCCESS; } @@ -2418,10 +2442,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } env->me_pglast = last; #if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u", last, txn->mt_dbs[FREE_DBI].md_root, i)); for (j = i; j; j--) - DPRINTF(("IDL %"Y"u", idl[j])); + DPRINTF(("IDL %"Yu, idl[j])); #endif /* Merge in descending sorted order */ mdb_midl_xmerge(mop, idl); @@ -2591,7 +2615,7 @@ mdb_page_touch(MDB_cursor *mc) (rc = mdb_page_alloc(mc, 1, &np))) goto fail; pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Y"u -> %"Y"u", DDBI(mc), + DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc), mp->mp_pgno, pgno)); mdb_cassert(mc, mp->mp_pgno != pgno); mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); @@ -2615,7 +2639,7 @@ mdb_page_touch(MDB_cursor *mc) if (mp != dl[x].mptr) { /* bad cursor? */ mc->mc_flags &= ~(C_INITIALIZED|C_EOF); txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } return 0; } @@ -2992,7 +3016,7 @@ mdb_txn_renew(MDB_txn *txn) rc = mdb_txn_renew0(txn); if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("renew txn %"Yu"%c %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3110,7 +3134,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) } else { txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ *ret = txn; - DPRINTF(("begin txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("begin txn %"Yu"%c %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3177,7 +3201,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) /* Export or close DBI handles opened in this txn */ mdb_dbis_update(txn, mode & MDB_END_UPDATE); - DPRINTF(("%s txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("%s txn %"Yu"%c %p on mdbenv %p, root page %"Yu, names[mode & MDB_END_OPMASK], txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); @@ -3378,10 +3402,10 @@ mdb_freelist_save(MDB_txn *txn) #if (MDB_DEBUG) > 1 { unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL write txn %"Yu" root %"Yu" num %u", txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); for (; i; i--) - DPRINTF(("IDL %"Y"u", free_pgs[i])); + DPRINTF(("IDL %"Yu, free_pgs[i])); } #endif continue; @@ -3549,7 +3573,7 @@ mdb_page_flush(MDB_txn *txn, int keep) * the write offset, to at least save the overhead of a Seek * system call. */ - DPRINTF(("committing page %"Z"u", pgno)); + DPRINTF(("committing page %"Yu, pgno)); memset(&ov, 0, sizeof(ov)); ov.Offset = pos & 0xffffffff; ov.OffsetHigh = pos >> 16 >> 16; @@ -3600,7 +3624,7 @@ mdb_page_flush(MDB_txn *txn, int keep) wpos = pos; wsize = 0; } - DPRINTF(("committing page %"Y"u", pgno)); + DPRINTF(("committing page %"Yu, pgno)); next_pos = pos + size; iov[n].iov_len = size; iov[n].iov_base = (char *)dp; @@ -3813,7 +3837,7 @@ mdb_txn_commit(MDB_txn *txn) !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) goto done; - DPRINTF(("committing txn %"Y"u %p on mdbenv %p, root page %"Y"u", + DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); /* Update DB root pointers */ @@ -3911,7 +3935,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta) p = (MDB_page *)&pbuf; if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Y"u not a meta page", p->mp_pgno)); + DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); return MDB_INVALID; } @@ -4023,7 +4047,7 @@ mdb_env_write_meta(MDB_txn *txn) #endif toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Y"u", + DPRINTF(("writing meta page %d for root page %"Yu, toggle, txn->mt_dbs[MAIN_DBI].md_root)); env = txn->mt_env; @@ -4521,13 +4545,13 @@ mdb_env_open2(MDB_env *env) DPRINTF(("opened database version %u, pagesize %u", meta->mm_version, env->me_psize)); - DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); - DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Y"u", db->md_entries)); - DPRINTF(("branch pages: %"Y"u", db->md_branch_pages)); - DPRINTF(("leaf pages: %"Y"u", db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Y"u", db->md_overflow_pages)); - DPRINTF(("root: %"Y"u", db->md_root)); + DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Yu, db->md_entries)); + DPRINTF(("branch pages: %"Yu, db->md_branch_pages)); + DPRINTF(("leaf pages: %"Yu, db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Yu, db->md_overflow_pages)); + DPRINTF(("root: %"Yu, db->md_root)); } #endif @@ -5417,7 +5441,7 @@ mdb_env_close(MDB_env *env) free(env); } -/** Compare two items pointing at aligned mdb_size_t's */ +/** Compare two items pointing at aligned #mdb_size_t's */ static int mdb_cmp_long(const MDB_val *a, const MDB_val *b) { @@ -5428,7 +5452,7 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b) /** Compare two items pointing at aligned unsigned int's. * * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, - * but #mdb_cmp_clong() is called instead if the data type is mdb_size_t. + * but #mdb_cmp_clong() is called instead if the data type is #mdb_size_t. */ static int mdb_cmp_int(const MDB_val *a, const MDB_val *b) @@ -5533,7 +5557,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) nkeys = NUMKEYS(mp); - DPRINTF(("searching %u keys in %s %spage %"Y"u", + DPRINTF(("searching %u keys in %s %spage %"Yu, nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp))); @@ -5581,7 +5605,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) DPRINTF(("found leaf index %u [%s], rc = %i", i, DKEY(&nodekey), rc)); else - DPRINTF(("found branch index %u [%s -> %"Y"u], rc = %i", + DPRINTF(("found branch index %u [%s -> %"Yu"], rc = %i", i, DKEY(&nodekey), NODEPGNO(node), rc)); #endif if (rc == 0) @@ -5629,7 +5653,7 @@ static void mdb_cursor_pop(MDB_cursor *mc) { if (mc->mc_snum) { - DPRINTF(("popping page %"Y"u off db %d cursor %p", + DPRINTF(("popping page %"Yu" off db %d cursor %p", mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); mc->mc_snum--; @@ -5645,7 +5669,7 @@ mdb_cursor_pop(MDB_cursor *mc) static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) { - DPRINTF(("pushing page %"Y"u on db %d cursor %p", mp->mp_pgno, + DPRINTF(("pushing page %"Yu" on db %d cursor %p", mp->mp_pgno, DDBI(mc), (void *) mc)); if (mc->mc_snum >= CURSOR_STACK) { @@ -5959,9 +5983,6 @@ static int mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) { MDB_txn *txn = mc->mc_txn; -#ifndef MDB_VL32 - MDB_env *env = txn->mt_env; -#endif MDB_page *p = NULL; int level; @@ -5980,14 +6001,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) MDB_ID pn = pgno << 1; x = mdb_midl_search(tx2->mt_spill_pgs, pn); if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { -#ifdef MDB_VL32 - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; -#else - p = (MDB_page *)(env->me_map + env->me_psize * pgno); -#endif - goto done; + goto mapped; } } if (dl[0].mid) { @@ -6001,21 +6015,24 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) } while ((tx2 = tx2->mt_parent) != NULL); } - if (pgno < txn->mt_next_pgno) { - level = 0; + if (pgno >= txn->mt_next_pgno) { + DPRINTF(("page %"Yu" not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + + level = 0; + +mapped: + { #ifdef MDB_VL32 - { - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; - } + int rc = mdb_rpage_get(txn, pgno, &p); + if (rc) + return rc; #else + MDB_env *env = txn->mt_env; p = (MDB_page *)(env->me_map + env->me_psize * pgno); #endif - } else { - DPRINTF(("page %"Y"u not found", pgno)); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_NOTFOUND; } done: @@ -6039,13 +6056,13 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) MDB_node *node; indx_t i; - DPRINTF(("branch page %"Y"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + DPRINTF(("branch page %"Yu" has %u keys", mp->mp_pgno, NUMKEYS(mp))); /* Don't assert on branch pages in the FreeDB. We can get here * while in the process of rebalancing a FreeDB branch page; we must * let that proceed. ITS#8336 */ mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Y"u", NODEPGNO(NODEPTR(mp, 0)))); + DPRINTF(("found index 0 to page %"Yu, NODEPGNO(NODEPTR(mp, 0)))); if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { i = 0; @@ -6090,7 +6107,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) return MDB_CORRUPTED; } - DPRINTF(("found leaf page %"Y"u for key [%s]", mp->mp_pgno, + DPRINTF(("found leaf page %"Yu" for key [%s]", mp->mp_pgno, key ? DKEY(key) : "null")); mc->mc_flags |= C_INITIALIZED; mc->mc_flags &= ~C_EOF; @@ -6205,7 +6222,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) mc->mc_snum = 1; mc->mc_top = 0; - DPRINTF(("db %d root page %"Y"u has flags 0x%X", + DPRINTF(("db %d root page %"Yu" has flags 0x%X", DDBI(mc), root, mc->mc_pg[0]->mp_flags)); if (flags & MDB_PS_MODIFY) { @@ -6230,7 +6247,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) MDB_ID pn = pg << 1; int rc; - DPRINTF(("free ov page %"Y"u (%d)", pg, ovpages)); + DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages)); /* If the page is dirty or on the spill list we just acquired it, * so we should give it back to our current free list, if any. * Otherwise put it onto the list of pages we freed in this txn. @@ -6271,7 +6288,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) j = ++(dl[0].mid); dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } } txn->mt_dirty_room++; @@ -6308,12 +6325,10 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) pgno_t pgno; int rc; -#ifdef MDB_VL32 - if (mc->mc_ovpg) { - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_ovpg); - mc->mc_ovpg = 0; + if (MC_OVPG(mc)) { + MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc)); + MC_SET_OVPG(mc, NULL); } -#endif if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { data->mv_size = NODEDSZ(leaf); data->mv_data = NODEDATA(leaf); @@ -6325,13 +6340,11 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) data->mv_size = NODEDSZ(leaf); memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Y"u failed", pgno)); + DPRINTF(("read overflow page %"Yu" failed", pgno)); return rc; } data->mv_data = METADATA(omp); -#ifdef MDB_VL32 - mc->mc_ovpg = omp; -#endif + MC_SET_OVPG(mc, omp); return MDB_SUCCESS; } @@ -6355,14 +6368,10 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi, mdb_cursor_init(&mc, txn, dbi, &mx); rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact); -#ifdef MDB_VL32 - { - /* unref all the pages - caller must copy the data - * before doing anything else - */ - mdb_cursor_unref(&mc); - } -#endif + /* unref all the pages when MDB_VL32 - caller must copy the data + * before doing anything else + */ + MDB_CURSOR_UNREF(&mc, 1); return rc; } @@ -6392,7 +6401,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right) op = mc->mc_pg[mc->mc_top]; #endif mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Y"u, index %u", + DPRINTF(("parent page is page %"Yu", index %u", mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) @@ -6459,13 +6468,9 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_NEXT_DUP) @@ -6473,7 +6478,7 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_next: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_next: top page is %"Yu" in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); if (mc->mc_flags & C_DEL) { mc->mc_flags ^= C_DEL; @@ -6487,12 +6492,12 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("next page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]++; skip: - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6552,13 +6557,9 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_PREV_DUP) @@ -6566,7 +6567,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_prev: top page is %"Yu" in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); mc->mc_flags &= ~(C_EOF|C_DEL); @@ -6578,13 +6579,13 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } mp = mc->mc_pg[mc->mc_top]; mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("prev page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]--; mc->mc_flags &= ~C_EOF; - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6627,8 +6628,10 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (key->mv_size == 0) return MDB_BAD_VALSIZE; - if (mc->mc_xcursor) + if (mc->mc_xcursor) { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } /* See if we're already on the right page */ if (mc->mc_flags & C_INITIALIZED) { @@ -6760,11 +6763,6 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, return MDB_SUCCESS; } -#ifdef MDB_VL32 - if (mc->mc_xcursor && mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { mdb_xcursor_init1(mc, leaf); } @@ -6790,10 +6788,8 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) return rc; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) dcmp = mdb_cmp_clong; -#endif rc = dcmp(data, &olddata); if (rc) { if (op == MDB_GET_BOTH || rc > 0) @@ -6826,11 +6822,7 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) MDB_node *leaf; if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); } @@ -6876,11 +6868,7 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) MDB_node *leaf; if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); } @@ -7336,10 +7324,8 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (flags == MDB_CURRENT) goto current; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) dcmp = mdb_cmp_clong; -#endif /* does data match? */ if (!dcmp(data, &olddata)) { if (flags & (MDB_NODUPDATA|MDB_APPENDDUP)) @@ -7664,7 +7650,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, return rc; bad_sub: if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ - rc = MDB_CORRUPTED; + rc = MDB_PROBLEM; } mc->mc_txn->mt_flags |= MDB_TXN_ERROR; return rc; @@ -7793,7 +7779,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) if ((rc = mdb_page_alloc(mc, num, &np))) return rc; - DPRINTF(("allocated new mpage %"Y"u, page size %u", + DPRINTF(("allocated new mpage %"Yu", page size %u", np->mp_pgno, mc->mc_txn->mt_env->me_psize)); np->mp_flags = flags | P_DIRTY; np->mp_lower = (PAGEHDRSZ-PAGEBASE); @@ -7893,7 +7879,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - DPRINTF(("add to %s %spage %"Y"u index %i, data size %"Z"u key size %"Z"u [%s]", + DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]", IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, @@ -7934,7 +7920,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, goto full; if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) return rc; - DPRINTF(("allocated overflow page %"Y"u", ofp->mp_pgno)); + DPRINTF(("allocated overflow page %"Yu, ofp->mp_pgno)); flags |= F_BIGDATA; goto update; } else { @@ -7991,7 +7977,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, return MDB_SUCCESS; full: - DPRINTF(("not enough room in page %"Y"u, got %u ptrs", + DPRINTF(("not enough room in page %"Yu", got %u ptrs", mdb_dbg_pgno(mp), NUMKEYS(mp))); DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); DPRINTF(("node size = %"Z"u", node_size)); @@ -8014,7 +8000,7 @@ mdb_node_del(MDB_cursor *mc, int ksize) MDB_node *node; char *base; - DPRINTF(("delete node %u on %s page %"Y"u", indx, + DPRINTF(("delete node %u on %s page %"Yu, indx, IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); numkeys = NUMKEYS(mp); mdb_cassert(mc, indx < numkeys); @@ -8123,9 +8109,7 @@ mdb_xcursor_init0(MDB_cursor *mc) mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; mx->mx_cursor.mc_snum = 0; mx->mx_cursor.mc_top = 0; -#ifdef MDB_VL32 - mx->mx_cursor.mc_ovpg = 0; -#endif + MC_SET_OVPG(&mx->mx_cursor, NULL); mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP)); mx->mx_dbx.md_name.mv_size = 0; mx->mx_dbx.md_name.mv_data = NULL; @@ -8173,13 +8157,11 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) mx->mx_db.md_flags |= MDB_INTEGERKEY; } } - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad)) mx->mx_dbx.md_cmp = mdb_cmp_clong; -#endif } @@ -8202,7 +8184,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) mx->mx_cursor.mc_flags |= C_INITIALIZED; mx->mx_cursor.mc_ki[0] = 0; mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX +#if UINT_MAX < MDB_SIZE_MAX /* matches mdb_xcursor_init1:NEED_CMP_CLONG() */ mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; #endif } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { @@ -8210,7 +8192,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) } mx->mx_db = src_mx->mx_db; mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); } @@ -8229,9 +8211,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) mc->mc_top = 0; mc->mc_pg[0] = 0; mc->mc_ki[0] = 0; -#ifdef MDB_VL32 - mc->mc_ovpg = 0; -#endif + MC_SET_OVPG(mc, NULL); mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { mdb_tassert(txn, mx != NULL); @@ -8331,6 +8311,9 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) void mdb_cursor_close(MDB_cursor *mc) { + if (mc) { + MDB_CURSOR_UNREF(mc, 0); + } if (mc && !mc->mc_backup) { /* remove from txn, if tracked */ if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { @@ -8382,7 +8365,7 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key) char kbuf2[DKBUF_MAXKEYSIZE*2+1]; k2.mv_data = NODEKEY(node); k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Y"u", + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Yu, indx, ptr, mdb_dkey(&k2, kbuf2), DKEY(key), @@ -8530,7 +8513,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) return rc; } - DPRINTF(("moving %s node %u [%s] on page %"Y"u to node %u on page %"Y"u", + DPRINTF(("moving %s node %u [%s] on page %"Yu" to node %u on page %"Yu, IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", csrc->mc_ki[csrc->mc_top], DKEY(&key), @@ -8624,7 +8607,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for source page %"Y"u to [%s]", + DPRINTF(("update separator for source page %"Yu" to [%s]", csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(csrc, &mn); mn.mc_snum--; @@ -8655,7 +8638,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for destination page %"Y"u to [%s]", + DPRINTF(("update separator for destination page %"Yu" to [%s]", cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(cdst, &mn); mn.mc_snum--; @@ -8701,7 +8684,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) psrc = csrc->mc_pg[csrc->mc_top]; pdst = cdst->mc_pg[cdst->mc_top]; - DPRINTF(("merging page %"Y"u into %"Y"u", psrc->mp_pgno, pdst->mp_pgno)); + DPRINTF(("merging page %"Yu" into %"Yu, psrc->mp_pgno, pdst->mp_pgno)); mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ mdb_cassert(csrc, cdst->mc_snum > 1); @@ -8758,7 +8741,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } } - DPRINTF(("dst page %"Y"u now has %u keys (%.1f%% filled)", + DPRINTF(("dst page %"Yu" now has %u keys (%.1f%% filled)", pdst->mp_pgno, NUMKEYS(pdst), (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); @@ -8846,9 +8829,7 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) cdst->mc_snum = csrc->mc_snum; cdst->mc_top = csrc->mc_top; cdst->mc_flags = csrc->mc_flags; -#ifdef MDB_VL32 - cdst->mc_ovpg = csrc->mc_ovpg; -#endif + MC_SET_OVPG(cdst, MC_OVPG(csrc)); for (i=0; imc_snum; i++) { cdst->mc_pg[i] = csrc->mc_pg[i]; @@ -8877,14 +8858,14 @@ mdb_rebalance(MDB_cursor *mc) minkeys = 1; thresh = FILL_THRESHOLD; } - DPRINTF(("rebalancing %s page %"Y"u (has %u keys, %.1f%% full)", + DPRINTF(("rebalancing %s page %"Yu" (has %u keys, %.1f%% full)", IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Y"u, above fill threshold", + DPRINTF(("no need to rebalance page %"Yu", above fill threshold", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); return MDB_SUCCESS; } @@ -9013,7 +8994,7 @@ mdb_rebalance(MDB_cursor *mc) fromleft = 1; } - DPRINTF(("found neighbor page %"Y"u (%u keys, %.1f%% full)", + DPRINTF(("found neighbor page %"Yu" (%u keys, %.1f%% full)", mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); @@ -9223,7 +9204,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno newindx = mc->mc_ki[mc->mc_top]; nkeys = NUMKEYS(mp); - DPRINTF(("-----> splitting %s page %"Y"u and adding [%s] at index %i/%i", + DPRINTF(("-----> splitting %s page %"Yu" and adding [%s] at index %i/%i", IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); @@ -9231,7 +9212,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) return rc; rp->mp_pad = mp->mp_pad; - DPRINTF(("new right sibling: page %"Y"u", rp->mp_pgno)); + DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno)); /* Usually when splitting the root page, the cursor * height is 1. But when called from mdb_update_key, @@ -9249,7 +9230,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mc->mc_pg[0] = pp; mc->mc_ki[0] = 0; mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Y"u", pp->mp_pgno)); + DPRINTF(("root split! new root = %"Yu, pp->mp_pgno)); new_root = mc->mc_db->md_depth++; /* Add left (implicit) pointer. */ @@ -9266,7 +9247,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno ptop = 0; } else { ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Y"u", mc->mc_pg[ptop]->mp_pgno)); + DPRINTF(("parent branch page is %"Yu, mc->mc_pg[ptop]->mp_pgno)); } mdb_cursor_copy(mc, &mn); @@ -10621,10 +10602,8 @@ mdb_drop0(MDB_cursor *mc, int subs) done: if (rc) txn->mt_flags |= MDB_TXN_ERROR; -#ifdef MDB_VL32 /* drop refcount for mx's pages */ - mdb_cursor_unref(&mx); -#endif + MDB_CURSOR_UNREF(&mx, 0); } else if (rc == MDB_NOTFOUND) { rc = MDB_SUCCESS; } @@ -10744,7 +10723,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) if (mr[i].mr_pid) { txnid_t txnid = mr[i].mr_txnid; sprintf(buf, txnid == (txnid_t)-1 ? - "%10d %"Z"x -\n" : "%10d %"Z"x %"Y"u\n", + "%10d %"Z"x -\n" : "%10d %"Z"x %"Yu"\n", (int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid); if (first) { first = 0; @@ -10849,7 +10828,7 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead) } for (; j -#include +#include "lmdb.h" #ifdef __cplusplus extern "C" { @@ -43,11 +42,7 @@ extern "C" { /** A generic unsigned ID number. These were entryIDs in back-bdb. * Preferably it should have the same size as a pointer. */ -#ifdef MDB_VL32 -typedef uint64_t MDB_ID; -#else -typedef size_t MDB_ID; -#endif +typedef mdb_size_t MDB_ID; /** An IDL is an ID List, a sorted array of IDs. The first * element of the array is a counter for how many actual From ecca82d7a2f01b988823bb3dff96c2fd07f8dcec Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Mon, 1 Aug 2016 14:06:06 +0200 Subject: [PATCH 02/15] Added Director profile operator This profile can be used by named consoles. It is intended to be used for daily administrative operations. It gives access to most of Bareos functionality, but blocking some potentially dangerous commands. --- debian/bareos-director.install.in | 1 + platforms/packaging/bareos.spec | 1 + .../bareos-dir.d/profile/operator.conf | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 src/defaultconfigs/bareos-dir.d/profile/operator.conf diff --git a/debian/bareos-director.install.in b/debian/bareos-director.install.in index 2c166a70f13..dac5cde56a1 100644 --- a/debian/bareos-director.install.in +++ b/debian/bareos-director.install.in @@ -22,6 +22,7 @@ @confdir@/bareos-dir.d/pool/Full.conf @configtemplatedir@/bareos-dir.d/pool/ @confdir@/bareos-dir.d/pool/Incremental.conf @configtemplatedir@/bareos-dir.d/pool/ @confdir@/bareos-dir.d/pool/Scratch.conf @configtemplatedir@/bareos-dir.d/pool/ +@confdir@/bareos-dir.d/profile/operator.conf @configtemplatedir@/bareos-dir.d/profile/ @confdir@/bareos-dir.d/schedule/WeeklyCycleAfterBackup.conf @configtemplatedir@/bareos-dir.d/schedule/ @confdir@/bareos-dir.d/schedule/WeeklyCycle.conf @configtemplatedir@/bareos-dir.d/schedule/ @confdir@/bareos-dir.d/storage/File.conf @configtemplatedir@/bareos-dir.d/storage/ diff --git a/platforms/packaging/bareos.spec b/platforms/packaging/bareos.spec index cd6093e56e6..79a2a427440 100644 --- a/platforms/packaging/bareos.spec +++ b/platforms/packaging/bareos.spec @@ -918,6 +918,7 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/pool/Full.conf %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/pool/Incremental.conf %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/pool/Scratch.conf +%attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/profile/operator.conf %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/schedule/WeeklyCycleAfterBackup.conf %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/schedule/WeeklyCycle.conf %attr(0640, %{director_daemon_user}, %{daemon_group}) %config(noreplace) %{_sysconfdir}/bareos/bareos-dir.d/storage/File.conf diff --git a/src/defaultconfigs/bareos-dir.d/profile/operator.conf b/src/defaultconfigs/bareos-dir.d/profile/operator.conf new file mode 100644 index 00000000000..1d1f4e91802 --- /dev/null +++ b/src/defaultconfigs/bareos-dir.d/profile/operator.conf @@ -0,0 +1,19 @@ +Profile { + Name = operator + Description = "Profile allowing normal Bareos operations." + + Command ACL = !.bvfs_clear_cache, !.exit, !.sql + Command ACL = !configure, !create, !delete, !purge, !sqlquery, !umount, !unmount + Command ACL = *all* + + Catalog ACL = *all* + Client ACL = *all* + FileSet ACL = *all* + Job ACL = *all* + Plugin Options ACL = *all* + Pool ACL = *all* + Schedule ACL = *all* + Storage ACL = *all* + Where ACL = *all* +} + From ef0e212dee3484a18f365861a8df8954aae4f76a Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Thu, 18 Aug 2016 13:17:11 +0200 Subject: [PATCH 03/15] Fix file name for exported resources An exported bareos-fd director resource needs to have the name of the director, and not of the client, for which it is generated. Also package the directory used for exporting client resources: /etc/bareos/bareos-dir-export/client/ --- debian/bareos-director.dirs | 1 + src/dird/Makefile.in | 2 +- src/dird/ua_configure.c | 19 +++++++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/debian/bareos-director.dirs b/debian/bareos-director.dirs index b2ef0f1e08b..e4c5e03af25 100644 --- a/debian/bareos-director.dirs +++ b/debian/bareos-director.dirs @@ -13,4 +13,5 @@ /etc/bareos/bareos-dir.d/schedule /etc/bareos/bareos-dir.d/storage /etc/bareos/bareos-dir-export/ +/etc/bareos/bareos-dir-export/client /usr/lib/bareos diff --git a/src/dird/Makefile.in b/src/dird/Makefile.in index 13096daf6ca..5696818c85d 100644 --- a/src/dird/Makefile.in +++ b/src/dird/Makefile.in @@ -170,7 +170,7 @@ install: all fi; \ done; \ done - $(MKDIR) ${DESTDIR}${confdir}/bareos-dir-export/bareos-fd.d/director/ + $(MKDIR) ${DESTDIR}${confdir}/bareos-dir-export/client/ @if test -f ${DESTDIR}${scriptdir}/query.sql; then \ echo " ==> Saving existing query.sql to query.sql.old"; \ $(MV) -f ${DESTDIR}${scriptdir}/query.sql ${DESTDIR}${scriptdir}/query.sql.old; \ diff --git a/src/dird/ua_configure.c b/src/dird/ua_configure.c index 42372083aa2..a1315f3be8d 100644 --- a/src/dird/ua_configure.c +++ b/src/dird/ua_configure.c @@ -216,6 +216,10 @@ static inline bool configure_create_fd_resource_string(POOL_MEM &resource, const return true; } +/* + * Create a bareos-fd director resource file + * that corresponds to our client definition. + */ static inline bool configure_create_fd_resource(UAContext *ua, const char *clientname) { POOL_MEM resource(PM_MESSAGE); @@ -223,6 +227,7 @@ static inline bool configure_create_fd_resource(UAContext *ua, const char *clien POOL_MEM filename(PM_FNAME); POOL_MEM basedir(PM_FNAME); POOL_MEM temp(PM_MESSAGE); + const char *dirname = NULL; const bool error_if_exists = false; const bool create_directories = true; @@ -230,15 +235,21 @@ static inline bool configure_create_fd_resource(UAContext *ua, const char *clien return false; } + /* + * Get the path where the resource should get stored. + */ basedir.bsprintf("bareos-dir-export/client/%s/bareos-fd.d", clientname); - if (!my_config->get_path_of_new_resource(filename, temp, basedir.c_str(), "director", clientname, - error_if_exists, create_directories)) { + dirname = GetNextRes(R_DIRECTOR, NULL)->name; + if (!my_config->get_path_of_new_resource(filename, temp, basedir.c_str(), "director", + dirname, error_if_exists, create_directories)) { ua->error_msg("%s", temp.c_str()); return false; - } else { - filename_tmp.strcpy(temp); } + filename_tmp.strcpy(temp); + /* + * Write resource to file. + */ if (!configure_write_resource(filename.c_str(), "filedaemon-export", clientname, resource.c_str())) { ua->error_msg("failed to write filedaemon config resource file\n"); return false; From cb52b40df955fb95487939aebe20b8aaf33d9bfa Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Thu, 18 Aug 2016 15:35:12 +0200 Subject: [PATCH 04/15] added "configure export client=" command Let the Bareos Director write the corresponding bareos-fd director resource for a configured client to /bareos-dir-export/client//bareos-fd.d/director/.conf --- src/dird/dird_conf.c | 14 +++++++---- src/dird/ua_configure.c | 52 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 016191300e0..b44c73c5179 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1019,19 +1019,23 @@ const char *get_configure_usage_string() * Only fill the configure_usage_string once. The content is static. */ if (configure_usage_string->strlen() == 0) { - configure_usage_string->strcpy("add ["); + /* + * subcommand: add + */ for (int r = 0; resources[r].name; r++) { if (resources[r].items) { + configure_usage_string->strcat("add "); resourcename.strcpy(resources[r].name); resourcename.toLower(); configure_usage_string->strcat(resourcename); cmdline_items(configure_usage_string, resources[r].items); - } - if (resources[r+1].items) { - configure_usage_string->strcat("] | ["); + configure_usage_string->strcat(" |\n"); } } - configure_usage_string->strcat("]"); + /* + * subcommand: export + */ + configure_usage_string->strcat("export client="); } return configure_usage_string->c_str(); diff --git a/src/dird/ua_configure.c b/src/dird/ua_configure.c index a1315f3be8d..192fdc7b293 100644 --- a/src/dird/ua_configure.c +++ b/src/dird/ua_configure.c @@ -230,6 +230,7 @@ static inline bool configure_create_fd_resource(UAContext *ua, const char *clien const char *dirname = NULL; const bool error_if_exists = false; const bool create_directories = true; + const bool overwrite = true; if (!configure_create_fd_resource_string(resource, clientname)) { return false; @@ -250,11 +251,20 @@ static inline bool configure_create_fd_resource(UAContext *ua, const char *clien /* * Write resource to file. */ - if (!configure_write_resource(filename.c_str(), "filedaemon-export", clientname, resource.c_str())) { + if (!configure_write_resource(filename.c_str(), "filedaemon-export", clientname, resource.c_str(), overwrite)) { ua->error_msg("failed to write filedaemon config resource file\n"); return false; } + ua->send->object_start("export"); + ua->send->object_key_value("clientname", clientname); + ua->send->object_key_value("component", "bareos-fd"); + ua->send->object_key_value("resource", "director"); + ua->send->object_key_value("name", dirname); + ua->send->object_key_value("filename", filename.c_str(), "Exported resource file \"%s\":\n"); + ua->send->object_key_value("content", resource.c_str(), "%s"); + ua->send->object_end("export"); + return true; } @@ -323,14 +333,12 @@ static inline bool configure_add_resource(UAContext *ua, int first_parameter, RE configure_create_fd_resource(ua, name.c_str()); } - ua->send->object_start("configure"); ua->send->object_start("add"); ua->send->object_key_value("resource", res_table->name); ua->send->object_key_value("name", name.c_str()); - ua->send->object_key_value("filename", filename.c_str(), "Created config file \"%s\":\n"); + ua->send->object_key_value("filename", filename.c_str(), "Created resource config file \"%s\":\n"); ua->send->object_key_value("content", resource.c_str(), "%s"); ua->send->object_end("add"); - ua->send->object_end("configure"); return true; } @@ -342,7 +350,9 @@ static inline bool configure_add(UAContext *ua, int resource_type_parameter) res_table = my_config->get_resource_table(ua->argk[resource_type_parameter]); if (res_table) { + ua->send->object_start("configure"); result = configure_add_resource(ua, resource_type_parameter+1, res_table); + ua->send->object_end("configure"); } else { ua->error_msg(_("invalid resource type %s.\n"), ua->argk[resource_type_parameter]); } @@ -350,6 +360,34 @@ static inline bool configure_add(UAContext *ua, int resource_type_parameter) return result; } +static inline void configure_export_usage(UAContext *ua) +{ + ua->error_msg(_("usage: configure export client=\n")); +} + +static inline bool configure_export(UAContext *ua) +{ + bool result = false; + int i; + + i = find_arg_with_value(ua, NT_("client")); + if (i < 0) { + configure_export_usage(ua); + return false; + } + + if (!GetClientResWithName(ua->argv[i])) { + configure_export_usage(ua); + return false; + } + + ua->send->object_start("configure"); + result = configure_create_fd_resource(ua, ua->argv[i]); + ua->send->object_end("configure"); + + return result; +} + bool configure_cmd(UAContext *ua, const char *cmd) { bool result = false; @@ -364,12 +402,16 @@ bool configure_cmd(UAContext *ua, const char *cmd) } if (ua->argc < 3) { - ua->error_msg(_("usage: configure ...\n")); + ua->error_msg(_("usage:\n" + " configure add = ...\n" + " configure export client=\n")); return false; } if (bstrcasecmp(ua->argk[1], NT_("add"))) { result = configure_add(ua, 2); + } else if (bstrcasecmp(ua->argk[1], NT_("export"))) { + result = configure_export(ua); } else { ua->error_msg(_("invalid subcommand %s.\n"), ua->argk[1]); return false; From ca944e3bf5acccd25f00e454840e79dbecb938f6 Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 19 Aug 2016 18:12:49 +0200 Subject: [PATCH 05/15] configure add: deny adding a director resource Only a single director resource is allowed per director. --- src/dird/dird_conf.c | 6 +++++- src/dird/ua_configure.c | 16 +++++++++++----- src/lib/parse_conf.c | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index b44c73c5179..4940af90aa5 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1023,7 +1023,11 @@ const char *get_configure_usage_string() * subcommand: add */ for (int r = 0; resources[r].name; r++) { - if (resources[r].items) { + /* + * Only one Director is allowed. + * If the resource have not items, there is no need to add it. + */ + if ((resources[r].rcode != R_DIRECTOR) && (resources[r].items)) { configure_usage_string->strcat("add "); resourcename.strcpy(resources[r].name); resourcename.toLower(); diff --git a/src/dird/ua_configure.c b/src/dird/ua_configure.c index 192fdc7b293..dfcf3f881b2 100644 --- a/src/dird/ua_configure.c +++ b/src/dird/ua_configure.c @@ -349,14 +349,20 @@ static inline bool configure_add(UAContext *ua, int resource_type_parameter) RES_TABLE *res_table = NULL; res_table = my_config->get_resource_table(ua->argk[resource_type_parameter]); - if (res_table) { - ua->send->object_start("configure"); - result = configure_add_resource(ua, resource_type_parameter+1, res_table); - ua->send->object_end("configure"); - } else { + if (!res_table) { ua->error_msg(_("invalid resource type %s.\n"), ua->argk[resource_type_parameter]); + return false; } + if (res_table->rcode == R_DIRECTOR) { + ua->error_msg(_("Only one Director resource allowed.\n")); + return false; + } + + ua->send->object_start("configure"); + result = configure_add_resource(ua, resource_type_parameter+1, res_table); + ua->send->object_end("configure"); + return result; } diff --git a/src/lib/parse_conf.c b/src/lib/parse_conf.c index 89a5e1d05b6..61c61176da0 100644 --- a/src/lib/parse_conf.c +++ b/src/lib/parse_conf.c @@ -169,7 +169,7 @@ bool CONFIG::parse_config_file(const char *cf, void *caller_ctx, LEX_ERROR_HANDL * Make two passes. The first builds the name symbol table, * and the second picks up the items. */ - Dmsg0(900, "Enter parse_config()\n"); + Dmsg1(900, "Enter parse_config_file(%s)\n", cf); for (pass = 1; pass <= 2; pass++) { Dmsg1(900, "parse_config pass %d\n", pass); if ((lc = lex_open_file(lc, cf, scan_error, scan_warning)) == NULL) { From b2ed8546d16758ce1684194e9a2985327a81685d Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Mon, 22 Aug 2016 18:12:11 +0200 Subject: [PATCH 06/15] Fix a problem with exiting on reload The CONFIG::parse_config_file can propagate exit behaviour on errors. It commit fixes a problem that occurs when using multiple configuration files. In this case, the default M_ERROR_TERM have been used in a couple of cases. --- src/lib/lex.c | 13 +++++++------ src/lib/parse_conf.c | 2 +- src/lib/protos.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/lex.c b/src/lib/lex.c index 90c4be28bc8..50e158e9c32 100644 --- a/src/lib/lex.c +++ b/src/lib/lex.c @@ -163,13 +163,14 @@ void lex_set_default_warning_handler(LEX *lf) /* * Set err_type used in error_handler - * return the old value */ -int lex_set_error_handler_error_type(LEX *lf, int err_type) +void lex_set_error_handler_error_type(LEX *lf, int err_type) { - int old = lf->err_type; - lf->err_type = err_type; - return old; + LEX *lex = lf; + while (lex) { + lex->err_type = err_type; + lex = lex->next; + } } /* @@ -255,7 +256,7 @@ static inline LEX *lex_add(LEX *lf, lf->options = nf->options; /* preserve user options */ /* * preserve err_type to prevent bareos exiting on 'reload' - * if config is invalid. Fixes bug #877 + * if config is invalid. */ lf->err_type = nf->err_type; } else { diff --git a/src/lib/parse_conf.c b/src/lib/parse_conf.c index 61c61176da0..8652c7b5d28 100644 --- a/src/lib/parse_conf.c +++ b/src/lib/parse_conf.c @@ -198,7 +198,7 @@ bool CONFIG::parse_config_file(const char *cf, void *caller_ctx, LEX_ERROR_HANDL cf, be.bstrerror()); free(lc); - return 0; + return false; } lex_set_error_handler_error_type(lc, err_type); lc->error_counter = 0; diff --git a/src/lib/protos.h b/src/lib/protos.h index b6f54f94fd4..d1506d6e130 100644 --- a/src/lib/protos.h +++ b/src/lib/protos.h @@ -279,7 +279,7 @@ const char *lex_tok_to_str(int token); int lex_get_token(LEX *lf, int expect); void lex_set_default_error_handler(LEX *lf); void lex_set_default_warning_handler(LEX *lf); -int lex_set_error_handler_error_type(LEX *lf, int err_type); +void lex_set_error_handler_error_type(LEX *lf, int err_type); /* message.c */ void my_name_is(int argc, char *argv[], const char *name); From e5f67abe1bdd516cdcee633c59c5aed9e223b895 Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 19 Aug 2016 18:54:09 +0200 Subject: [PATCH 07/15] show cmd: add verbose option When using the verbose option, also default and inherit directives are show. These directives are prefixed by a "# " character. --- src/console/console_conf.c | 4 +- src/dird/dird_conf.c | 64 ++++---- src/dird/dird_conf.h | 2 +- src/dird/job.c | 2 +- src/dird/ua_cmds.c | 6 +- src/dird/ua_output.c | 16 +- src/filed/filed_conf.c | 4 +- src/lib/parse_conf.h | 13 +- src/lib/res.c | 242 ++++++++++++++++-------------- src/qt-console/bat_conf.cpp | 4 +- src/qt-tray-monitor/tray_conf.cpp | 4 +- src/stored/stored_conf.c | 4 +- 12 files changed, 195 insertions(+), 170 deletions(-) diff --git a/src/console/console_conf.c b/src/console/console_conf.c index abb8eb02432..30cfb631564 100644 --- a/src/console/console_conf.c +++ b/src/console/console_conf.c @@ -110,7 +110,7 @@ static RES_TABLE resources[] = { */ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { POOL_MEM buf; URES *res = (URES *)reshdr; @@ -135,7 +135,7 @@ void dump_resource(int type, RES *reshdr, sendit(sock, "%s", buf.c_str()); if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 4940af90aa5..1509010c31e 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1295,14 +1295,6 @@ char *CATRES::display(POOLMEM *dst) return dst; } -static void indent_config_item(POOL_MEM &cfg_str, int level, const char *config_item) -{ - for (int i = 0; i < level; i++) { - pm_strcat(cfg_str, DEFAULT_INDENT_STRING); - } - pm_strcat(cfg_str, config_item); -} - static inline void print_config_runscript(RES_ITEM *item, POOL_MEM &cfg_str) { POOL_MEM temp; @@ -1856,7 +1848,7 @@ static inline void print_config_run(RES_ITEM *item, POOL_MEM &cfg_str) } } -bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) +bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data, bool verbose) { POOL_MEM cfg_str; POOL_MEM temp; @@ -2277,7 +2269,7 @@ const char *level_to_str(int level) */ void dump_resource(int type, RES *ures, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { URES *res = (URES *)ures; bool recurse = true; @@ -2296,71 +2288,71 @@ void dump_resource(int type, RES *ures, switch (type) { case R_DIRECTOR: - res->res_dir.print_config(buf, hide_sensitive_data); + res->res_dir.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; case R_PROFILE: - res->res_profile.print_config(buf, hide_sensitive_data); + res->res_profile.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; case R_CONSOLE: - res->res_con.print_config(buf, hide_sensitive_data); + res->res_con.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; case R_COUNTER: - res->res_counter.print_config(buf, hide_sensitive_data); + res->res_counter.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; case R_CLIENT: if (!ua || acl_access_ok(ua, Client_ACL, res->res_client.name())) { - res->res_client.print_config(buf, hide_sensitive_data); + res->res_client.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_DEVICE: - res->res_dev.print_config(buf, hide_sensitive_data); + res->res_dev.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; case R_STORAGE: if (!ua || acl_access_ok(ua, Storage_ACL, res->res_store.name())) { - res->res_store.print_config(buf, hide_sensitive_data); + res->res_store.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_CATALOG: if (!ua || acl_access_ok(ua, Catalog_ACL, res->res_cat.name())) { - res->res_cat.print_config(buf, hide_sensitive_data); + res->res_cat.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_JOBDEFS: case R_JOB: if (!ua || acl_access_ok(ua, Job_ACL, res->res_job.name())) { - res->res_job.print_config(buf, hide_sensitive_data); + res->res_job.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_FILESET: { if (!ua || acl_access_ok(ua, FileSet_ACL, res->res_fs.name())) { - res->res_fs.print_config(buf, hide_sensitive_data); + res->res_fs.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; } case R_SCHEDULE: if (!ua || acl_access_ok(ua, Schedule_ACL, res->res_sch.name())) { - res->res_sch.print_config(buf, hide_sensitive_data); + res->res_sch.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_POOL: if (!ua || acl_access_ok(ua, Pool_ACL, res->res_pool.name())) { - res->res_pool.print_config(buf, hide_sensitive_data); + res->res_pool.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); } break; case R_MSGS: - res->res_msgs.print_config(buf, hide_sensitive_data); + res->res_msgs.print_config(buf, hide_sensitive_data, verbose); sendit(sock, "%s", buf.c_str()); break; default: @@ -2369,7 +2361,7 @@ void dump_resource(int type, RES *ures, } if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } @@ -3981,7 +3973,7 @@ static void parse_config_cb(LEX *lc, RES_ITEM *item, int index, int pass) * callback function for print_config * See ../lib/res.c, function BRSRES::print_config, for more generic handling. */ -static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide_sensitive_data) +static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide_sensitive_data, bool inherited) { POOL_MEM temp; @@ -3998,7 +3990,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide list = *(items[i].alistvalue); if (list != NULL) { Mmsg(temp, "%s = ", items[i].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); pm_strcpy(res_names, ""); foreach_alist(res, list) { @@ -4034,7 +4026,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide list = items[i].alistvalue[items[i].code]; if (list != NULL) { Mmsg(temp, "%s = ", items[i].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); foreach_alist(value, list) { if (cnt) { Mmsg(temp, ",\"%s\"", value); @@ -4060,7 +4052,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; jobtypes[j].type_name; j++) { if (jobtypes[j].job_type == jobtype) { Mmsg(temp, "%s = %s\n", items[i].name, jobtypes[j].type_name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4083,7 +4075,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide } Mmsg(temp, "%s = %s\n", items[i].name, backupprotocols[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4097,7 +4089,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; migtypes[j].type_name; j++) { if (migtypes[j].job_type == migtype) { Mmsg(temp, "%s = %s\n", items[i].name, migtypes[j].type_name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4120,7 +4112,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide } Mmsg(temp, "%s = %s\n", items[i].name, ReplaceOptions[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4134,7 +4126,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; joblevels[j].level_name; j++) { if (joblevels[j].level == level) { Mmsg(temp, "%s = %s\n", items[i].name, joblevels[j].level_name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4148,7 +4140,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; ActionOnPurgeOptions[j].name; j++) { if (ActionOnPurgeOptions[j].token == action) { Mmsg(temp, "%s = %s\n", items[i].name, ActionOnPurgeOptions[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4162,7 +4154,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; authprotocols[j].name; j++) { if (authprotocols[j].token == authprotocol) { Mmsg(temp, "%s = %s\n", items[i].name, authprotocols[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4176,7 +4168,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide for (int j = 0; authmethods[j].name; j++) { if (authprotocols[j].token == authtype) { Mmsg(temp, "%s = %s\n", items[i].name, authmethods[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -4195,7 +4187,7 @@ static void print_config_cb(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide list = *(items[i].alistvalue); if (list != NULL) { Mmsg(temp, "%s = ", items[i].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); pm_strcpy(audit_events, ""); foreach_alist(audit_event, list) { diff --git a/src/dird/dird_conf.h b/src/dird/dird_conf.h index 84507dbc0a9..6f589452ab5 100644 --- a/src/dird/dird_conf.h +++ b/src/dird/dird_conf.h @@ -498,7 +498,7 @@ class FILESETRES : public BRSRES { bool enable_vss; /* Enable Volume Shadow Copy */ /* Methods */ - bool print_config(POOL_MEM& buff, bool hide_sensitive_data); + bool print_config(POOL_MEM& buf, bool hide_sensitive_data = false, bool verbose = false); }; /* diff --git a/src/dird/job.c b/src/dird/job.c index 3f770ebb133..78eb6f03d27 100644 --- a/src/dird/job.c +++ b/src/dird/job.c @@ -1347,7 +1347,7 @@ bool get_or_create_fileset_record(JCR *jcr) !db_get_fileset_record(jcr, jcr->db, &fsr)) { POOL_MEM FileSetText(PM_MESSAGE); - jcr->res.fileset->print_config(FileSetText, false); + jcr->res.fileset->print_config(FileSetText, false, false); fsr.FileSetText = FileSetText.c_str(); if (!db_create_fileset_record(jcr, jcr->db, &fsr)) { diff --git a/src/dird/ua_cmds.c b/src/dird/ua_cmds.c index 74d235f57b0..8520985f51a 100644 --- a/src/dird/ua_cmds.c +++ b/src/dird/ua_cmds.c @@ -373,9 +373,9 @@ static struct cmdstruct commands[] = { NT_(""), false, true }, { NT_("show"), show_cmd, _("Show resource records"), NT_("jobdefs= | job= | pool= | fileset= |\n" - "\tschedule= | client= | message= |\n" - "\tprofile= | jobdefs | jobs | pools | filesets | schedules | clients |\n" - "\tmessages | profiles | consoles | disabled | all"), true, true }, + "schedule= | client= | message= |\n" + "profile= | jobdefs | jobs | pools | filesets | schedules | clients |\n" + "messages | profiles | consoles | disabled [ clients | jobs | schedules ] | all [verbose]"), true, true }, { NT_("sqlquery"), sqlquery_cmd, _("Use SQL to query catalog"), NT_(""), false, true }, { NT_("time"), time_cmd, _("Print current time"), diff --git a/src/dird/ua_output.c b/src/dird/ua_output.c index da129e080b1..c431e9fd858 100644 --- a/src/dird/ua_output.c +++ b/src/dird/ua_output.c @@ -223,6 +223,7 @@ bool show_cmd(UAContext *ua, const char *cmd) int recurse; char *res_name; RES *res = NULL; + bool verbose = false; bool hide_sensitive_data; Dmsg1(20, "show: %s\n", ua->UA_sock->msg); @@ -233,8 +234,19 @@ bool show_cmd(UAContext *ua, const char *cmd) */ hide_sensitive_data = !acl_access_ok(ua, Command_ACL, "configure", false); + if (find_arg(ua, NT_("verbose")) > 0) { + verbose = true; + } + LockRes(); for (i = 1; i < ua->argc; i++) { + /* + * skip verbose keyword, already handled earlier. + */ + if (bstrcasecmp(ua->argk[i], NT_("verbose"))) { + continue; + } + if (bstrcasecmp(ua->argk[i], _("disabled"))) { if (((i + 1) < ua->argc) && bstrcasecmp(ua->argk[i + 1], NT_("jobs"))) { @@ -303,7 +315,7 @@ bool show_cmd(UAContext *ua, const char *cmd) default: if (my_config->m_res_head[j - my_config->m_r_first]) { dump_resource(j, my_config->m_res_head[j - my_config->m_r_first], - bsendmsg, ua, hide_sensitive_data); + bsendmsg, ua, hide_sensitive_data, verbose); } break; } @@ -322,7 +334,7 @@ bool show_cmd(UAContext *ua, const char *cmd) ua->error_msg(_("Resource %s not found\n"), res_name); goto bail_out; default: - dump_resource(recurse ? type : -type, res, bsendmsg, ua, hide_sensitive_data); + dump_resource(recurse ? type : -type, res, bsendmsg, ua, hide_sensitive_data, verbose); break; } } diff --git a/src/filed/filed_conf.c b/src/filed/filed_conf.c index 99a895f7144..94a6127ac00 100644 --- a/src/filed/filed_conf.c +++ b/src/filed/filed_conf.c @@ -164,7 +164,7 @@ static RES_TABLE resources[] = { */ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { POOL_MEM buf; URES *res = (URES *)reshdr; @@ -195,7 +195,7 @@ void dump_resource(int type, RES *reshdr, sendit(sock, "%s", buf.c_str()); if (recurse && res->res_dir.hdr.next) { - dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, res->res_dir.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } diff --git a/src/lib/parse_conf.h b/src/lib/parse_conf.h index 159e5a7eed3..e34aaaa5da1 100644 --- a/src/lib/parse_conf.h +++ b/src/lib/parse_conf.h @@ -330,6 +330,11 @@ class BRSRES { /* Methods */ char *name() const; bool print_config(POOL_MEM &buf, bool hide_sensitive_data = false); + /* + * validate can be defined by inherited classes, + * when special rules for this resource type must be checked. + */ + bool validate(); }; inline char *BRSRES::name() const { return this->hdr.name; } @@ -366,12 +371,12 @@ class MSGSRES : public BRSRES { void wait_not_in_use(); /* in message.c */ void lock(); /* in message.c */ void unlock(); /* in message.c */ - bool print_config(POOL_MEM &buff, bool hide_sensitive_data = false); + bool print_config(POOL_MEM &buff, bool hide_sensitive_data = false, bool verbose = false); }; typedef void (INIT_RES_HANDLER)(RES_ITEM *item, int pass); typedef void (STORE_RES_HANDLER)(LEX *lc, RES_ITEM *item, int index, int pass); -typedef void (PRINT_RES_HANDLER)(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide_sensitive_data); +typedef void (PRINT_RES_HANDLER)(RES_ITEM *items, int i, POOL_MEM &cfg_str, bool hide_sensitive_data, bool inherited); /* * New C++ configuration routines @@ -473,13 +478,15 @@ RES *GetNextRes(int rcode, RES *res); void b_LockRes(const char *file, int line); void b_UnlockRes(const char *file, int line); void dump_resource(int type, RES *res, void sendmsg(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data = false); + void *sock, bool hide_sensitive_data = false, bool verbose = false); +void indent_config_item(POOL_MEM &cfg_str, int level, const char *config_item, bool inherited = false); void free_resource(RES *res, int type); void init_resource(int type, RES_ITEM *item); bool save_resource(int type, RES_ITEM *item, int pass); bool store_resource(int type, LEX *lc, RES_ITEM *item, int index, int pass); const char *res_to_str(int rcode); + #ifdef HAVE_JANSSON /* * JSON output helper functions diff --git a/src/lib/res.c b/src/lib/res.c index c4c7e31f0dd..1c0950007c8 100644 --- a/src/lib/res.c +++ b/src/lib/res.c @@ -1305,17 +1305,19 @@ bool store_resource(int type, LEX *lc, RES_ITEM *item, int index, int pass) return true; } -static void indent_config_item(POOL_MEM &cfg_str, int level, const char *config_item) +void indent_config_item(POOL_MEM &cfg_str, int level, const char *config_item, bool inherited) { - int i; - - for (i = 0; i < level; i++) { + for (int i = 0; i < level; i++) { + pm_strcat(cfg_str, DEFAULT_INDENT_STRING); + } + if (inherited) { + pm_strcat(cfg_str, "#"); pm_strcat(cfg_str, DEFAULT_INDENT_STRING); } pm_strcat(cfg_str, config_item); } -static inline void print_config_size(RES_ITEM *item, POOL_MEM &cfg_str) +static inline void print_config_size(RES_ITEM *item, POOL_MEM &cfg_str, bool inherited) { POOL_MEM temp; POOL_MEM volspec; /* vol specification string*/ @@ -1358,10 +1360,10 @@ static inline void print_config_size(RES_ITEM *item, POOL_MEM &cfg_str) } Mmsg(temp, "%s = %s\n", item->name, volspec.c_str()); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } -static inline void print_config_time(RES_ITEM *item, POOL_MEM &cfg_str) +static inline void print_config_time(RES_ITEM *item, POOL_MEM &cfg_str, bool inherited) { POOL_MEM temp; POOL_MEM timespec; @@ -1411,10 +1413,10 @@ static inline void print_config_time(RES_ITEM *item, POOL_MEM &cfg_str) } Mmsg(temp, "%s = %s\n", item->name, timespec.c_str()); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } -bool MSGSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) +bool MSGSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data, bool verbose) { POOL_MEM cfg_str; /* configuration as string */ POOL_MEM temp; @@ -1496,162 +1498,174 @@ bool MSGSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) return true; } -bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) +bool has_default_value(RES_ITEM *item) { - POOL_MEM cfg_str; - POOL_MEM temp; - RES_ITEM *items; - int i = 0; - int rindex; + bool is_default = false; - /* - * If entry is not used, then there is nothing to print. - */ - if (this->hdr.rcode < (uint32_t)my_config->m_r_first || - this->hdr.refcnt <= 0) { - return true; - } - rindex = this->hdr.rcode - my_config->m_r_first; - - /* - * Make sure the resource class has any items. - */ - if (!my_config->m_resources[rindex].items) { - return true; - } - - memcpy(my_config->m_res_all, this, my_config->m_resources[rindex].size); - - pm_strcat(cfg_str, res_to_str(this->hdr.rcode)); - pm_strcat(cfg_str, " {\n"); - - items = my_config->m_resources[rindex].items; - - for (i = 0; items[i].name; i++) { - bool print_item = false; - - /* - * If this is an alias for another config keyword suppress it. - */ - if ((items[i].flags & CFG_ITEM_ALIAS)) { - continue; - } - - if ((items[i].flags & CFG_ITEM_REQUIRED) || !my_config->m_omit_defaults) { - /* - * Always print required items or if my_config->m_omit_defaults is false - */ - print_item = true; - } else if (items[i].flags & CFG_ITEM_DEFAULT) { - /* - * See if the item has inherited content from somewhere. - * If that is true there is no need to check the whole default setting. - */ - if (!bit_is_set(i, this->hdr.inherit_content)) { + if (item->flags & CFG_ITEM_DEFAULT) { /* * Check for default values. */ - switch (items[i].type) { + switch (item->type) { case CFG_TYPE_STR: case CFG_TYPE_DIR: case CFG_TYPE_NAME: case CFG_TYPE_STRNAME: - print_item = !bstrcmp(*(items[i].value), items[i].default_value); + is_default = bstrcmp(*(item->value), item->default_value); break; case CFG_TYPE_INT16: - print_item = (*(items[i].i16value) != (int16_t)str_to_int32(items[i].default_value)); + is_default = (*(item->i16value) == (int16_t)str_to_int32(item->default_value)); break; case CFG_TYPE_PINT16: - print_item = (*(items[i].ui16value) != (uint16_t)str_to_int32(items[i].default_value)); + is_default = (*(item->ui16value) == (uint16_t)str_to_int32(item->default_value)); break; case CFG_TYPE_INT32: - print_item = (*(items[i].i32value) != str_to_int32(items[i].default_value)); + is_default = (*(item->i32value) == str_to_int32(item->default_value)); break; case CFG_TYPE_PINT32: - print_item = (*(items[i].ui32value) != (uint32_t)str_to_int32(items[i].default_value)); + is_default = (*(item->ui32value) == (uint32_t)str_to_int32(item->default_value)); break; case CFG_TYPE_INT64: - print_item = (*(items[i].i64value) != str_to_int64(items[i].default_value)); + is_default = (*(item->i64value) == str_to_int64(item->default_value)); break; case CFG_TYPE_SPEED: - print_item = (*(items[i].ui64value) != (uint64_t)str_to_int64(items[i].default_value)); + is_default = (*(item->ui64value) == (uint64_t)str_to_int64(item->default_value)); break; case CFG_TYPE_SIZE64: - print_item = (*(items[i].ui64value) != (uint64_t)str_to_int64(items[i].default_value)); + is_default = (*(item->ui64value) == (uint64_t)str_to_int64(item->default_value)); break; case CFG_TYPE_SIZE32: - print_item = (*(items[i].ui32value) != (uint32_t)str_to_int32(items[i].default_value)); + is_default = (*(item->ui32value) != (uint32_t)str_to_int32(item->default_value)); break; case CFG_TYPE_TIME: - print_item = (*(items[i].ui64value) != (uint64_t)str_to_int64(items[i].default_value)); + is_default = (*(item->ui64value) == (uint64_t)str_to_int64(item->default_value)); break; case CFG_TYPE_BOOL: { - bool default_value = bstrcasecmp(items[i].default_value, "true") || - bstrcasecmp(items[i].default_value, "yes"); + bool default_value = bstrcasecmp(item->default_value, "true") || + bstrcasecmp(item->default_value, "yes"); - print_item = (*items[i].boolvalue != default_value); + is_default = (*item->boolvalue == default_value); break; } default: break; } - } - } else { - /* - * See if the item has inherited content from somewhere. - * If that is true there is no need to check the whole default setting. - */ - if (!bit_is_set(i, this->hdr.inherit_content)) { - switch (items[i].type) { + } else { + switch (item->type) { case CFG_TYPE_STR: case CFG_TYPE_DIR: case CFG_TYPE_NAME: case CFG_TYPE_STRNAME: - print_item = *(items[i].value) != NULL; + is_default = (*(item->value) == NULL); break; case CFG_TYPE_INT16: - print_item = (*(items[i].i16value) > 0); + is_default = (*(item->i16value) == 0); break; case CFG_TYPE_PINT16: - print_item = (*(items[i].ui16value) > 0); + is_default = (*(item->ui16value) == 0); break; case CFG_TYPE_INT32: - print_item = (*(items[i].i32value) > 0); + is_default = (*(item->i32value) == 0); break; case CFG_TYPE_PINT32: - print_item = (*(items[i].ui32value) > 0); + is_default = (*(item->ui32value) == 0); break; case CFG_TYPE_INT64: - print_item = (*(items[i].i64value) > 0); + is_default = (*(item->i64value) == 0); break; case CFG_TYPE_SPEED: - print_item = (*(items[i].ui64value) > 0); + is_default = (*(item->ui64value) == 0); break; case CFG_TYPE_SIZE64: - print_item = (*(items[i].ui64value) > 0); + is_default = (*(item->ui64value) == 0); break; case CFG_TYPE_SIZE32: - print_item = (*(items[i].ui32value) > 0); + is_default = (*(item->ui32value) == 0); break; case CFG_TYPE_TIME: - print_item = (*(items[i].ui64value) > 0); + is_default = (*(item->ui64value) == 0); break; case CFG_TYPE_BOOL: - print_item = (*items[i].boolvalue != false); + is_default = (*item->boolvalue == false); break; default: break; } - } - } + } + + return is_default; +} + +bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data, bool verbose) +{ + POOL_MEM cfg_str; + POOL_MEM temp; + RES_ITEM *items; + int i = 0; + int rindex; + bool inherited = false; + + /* + * If entry is not used, then there is nothing to print. + */ + if (this->hdr.rcode < (uint32_t)my_config->m_r_first || + this->hdr.refcnt <= 0) { + return true; + } + rindex = this->hdr.rcode - my_config->m_r_first; + + /* + * Make sure the resource class has any items. + */ + if (!my_config->m_resources[rindex].items) { + return true; + } + + memcpy(my_config->m_res_all, this, my_config->m_resources[rindex].size); + + pm_strcat(cfg_str, res_to_str(this->hdr.rcode)); + pm_strcat(cfg_str, " {\n"); + + items = my_config->m_resources[rindex].items; + + for (i = 0; items[i].name; i++) { + bool print_item = false; + inherited = bit_is_set(i, this->hdr.inherit_content); /* - * See if the item has inherited content from somewhere. + * If this is an alias for another config keyword suppress it. */ - if (bit_is_set(i, this->hdr.inherit_content)) { + if ((items[i].flags & CFG_ITEM_ALIAS)) { + continue; + } + + if ((!verbose) && inherited) { + /* + * If not in verbose mode, skip inherited directives. + */ continue; } + if ((items[i].flags & CFG_ITEM_REQUIRED) || !my_config->m_omit_defaults) { + /* + * Always print required items or if my_config->m_omit_defaults is false + */ + print_item = true; + } + + if (!has_default_value(&items[i])) { + print_item = true; + } else { + if ((items[i].flags & CFG_ITEM_DEFAULT) && verbose) { + /* + * If value has a expliciet default value and verbose mode is on, + * display directive as inherited. + */ + print_item = true; + inherited = true; + } + } + switch (items[i].type) { case CFG_TYPE_STR: case CFG_TYPE_DIR: @@ -1660,7 +1674,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) if (print_item && *(items[i].value) != NULL) { Dmsg2(200, "%s = \"%s\"\n", items[i].name, *(items[i].value)); Mmsg(temp, "%s = \"%s\"\n", items[i].name, *(items[i].value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_MD5PASSWORD: @@ -1688,7 +1702,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) break; } } - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } } break; @@ -1705,7 +1719,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } Mmsg(temp, "%s = \"%s\"\n", items[i].name, tapelabels[j].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; } } @@ -1713,48 +1727,48 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) case CFG_TYPE_INT16: if (print_item) { Mmsg(temp, "%s = %hd\n", items[i].name, *(items[i].i16value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_PINT16: if (print_item) { Mmsg(temp, "%s = %hu\n", items[i].name, *(items[i].ui16value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_INT32: if (print_item) { Mmsg(temp, "%s = %d\n", items[i].name, *(items[i].i32value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_PINT32: if (print_item) { Mmsg(temp, "%s = %u\n", items[i].name, *(items[i].ui32value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_INT64: if (print_item) { Mmsg(temp, "%s = %d\n", items[i].name, *(items[i].i64value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_SPEED: if (print_item) { Mmsg(temp, "%s = %u\n", items[i].name, *(items[i].ui64value)); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_SIZE64: case CFG_TYPE_SIZE32: if (print_item) { - print_config_size(&items[i], cfg_str); + print_config_size(&items[i], cfg_str, inherited); } break; case CFG_TYPE_TIME: if (print_item) { - print_config_time(&items[i], cfg_str); + print_config_time(&items[i], cfg_str, inherited); } break; case CFG_TYPE_BOOL: @@ -1764,7 +1778,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } else { Mmsg(temp, "%s = %s\n", items[i].name, NT_("no")); } - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; case CFG_TYPE_ALIST_STR: @@ -1788,7 +1802,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } } Mmsg(temp, "%s = \"%s\"\n", items[i].name, value); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } } break; @@ -1805,7 +1819,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) list = *(items[i].alistvalue); if (list != NULL) { Mmsg(temp, "%s = ", items[i].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); pm_strcpy(res_names, ""); foreach_alist(res, list) { @@ -1829,7 +1843,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) res = *(items[i].resvalue); if (res != NULL && res->name != NULL) { Mmsg(temp, "%s = \"%s\"\n", items[i].name, res->name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); } break; } @@ -1839,7 +1853,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } else { Mmsg(temp, "%s = %s\n", items[i].name, NT_("no")); } - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); break; case CFG_TYPE_MSGS: /* @@ -1851,7 +1865,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) IPADDR *adr; Mmsg(temp, "%s = {\n", items[i].name); - indent_config_item(cfg_str, 1, temp.c_str()); + indent_config_item(cfg_str, 1, temp.c_str(), inherited); foreach_dlist(adr, addrs) { char tmp[1024]; @@ -1878,7 +1892,7 @@ bool BRSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) * This is a non-generic type call back to the daemon to get things printed. */ if (my_config->m_print_res) { - my_config->m_print_res(items, i, cfg_str, hide_sensitive_data); + my_config->m_print_res(items, i, cfg_str, hide_sensitive_data, inherited); } break; } diff --git a/src/qt-console/bat_conf.cpp b/src/qt-console/bat_conf.cpp index 48f063cff0e..7cb3d30363b 100644 --- a/src/qt-console/bat_conf.cpp +++ b/src/qt-console/bat_conf.cpp @@ -114,7 +114,7 @@ static RES_TABLE resources[] = { */ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { URES *res = (URES *)reshdr; bool recurse = true; @@ -143,7 +143,7 @@ void dump_resource(int type, RES *reshdr, printf(_("Unknown resource type %d\n"), type); } if (recurse && res->dir_res.hdr.next) { - dump_resource(type, res->dir_res.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, res->dir_res.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } diff --git a/src/qt-tray-monitor/tray_conf.cpp b/src/qt-tray-monitor/tray_conf.cpp index 4be0ccf12f0..2aad8a0510b 100644 --- a/src/qt-tray-monitor/tray_conf.cpp +++ b/src/qt-tray-monitor/tray_conf.cpp @@ -165,7 +165,7 @@ static RES_TABLE resources[] = { * Dump contents of resource */ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { POOL_MEM buf; URES *res = (URES *)reshdr; @@ -189,7 +189,7 @@ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fm sendit(sock, "%s", buf.c_str()); if (recurse && res->res_monitor.hdr.next) { - dump_resource(type, res->res_monitor.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, res->res_monitor.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } diff --git a/src/stored/stored_conf.c b/src/stored/stored_conf.c index 1afab13bae8..4676a1847c6 100644 --- a/src/stored/stored_conf.c +++ b/src/stored/stored_conf.c @@ -432,7 +432,7 @@ static void store_compressionalgorithm(LEX *lc, RES_ITEM *item, int index, int p */ void dump_resource(int type, RES *reshdr, void sendit(void *sock, const char *fmt, ...), - void *sock, bool hide_sensitive_data) + void *sock, bool hide_sensitive_data, bool verbose) { POOL_MEM buf; URES *res = (URES *)reshdr; @@ -463,7 +463,7 @@ void dump_resource(int type, RES *reshdr, sendit(sock, "%s", buf.c_str()); if (recurse && res->res_dir.hdr.next) { - dump_resource(type, (RES *)res->res_dir.hdr.next, sendit, sock, hide_sensitive_data); + dump_resource(type, (RES *)res->res_dir.hdr.next, sendit, sock, hide_sensitive_data, verbose); } } From bcbb17f64273891efa924f8865859d2f9d2b347a Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Wed, 24 Aug 2016 23:20:05 +0200 Subject: [PATCH 08/15] subdirectory configuration scheme: adapt usage text --- src/console/console.c | 2 +- src/dird/dird.c | 4 ++-- src/stored/bcopy.c | 2 +- src/stored/bextract.c | 2 +- src/stored/bls.c | 2 +- src/stored/bscan.c | 2 +- src/stored/btape.c | 2 +- src/stored/stored.c | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/console/console.c b/src/console/console.c index b50ca3d0ba7..dc47cdb8696 100644 --- a/src/console/console.c +++ b/src/console/console.c @@ -115,7 +115,7 @@ PROG_COPYRIGHT "Usage: bconsole [-s] [-c config_file] [-d debug_level]\n" " -D select a Director\n" " -l list Directors defined\n" -" -c set configuration file to file\n" +" -c specify configuration file or directory\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" " -n no conio\n" diff --git a/src/dird/dird.c b/src/dird/dird.c index 8e714b6ae50..ecfba0073b7 100644 --- a/src/dird/dird.c +++ b/src/dird/dird.c @@ -169,8 +169,8 @@ static void usage() fprintf(stderr, _( PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" -"Usage: bareos-dir [options] [-c config_file] [-d debug_level] [config_file]\n" -" -c use as configuration file\n" +"Usage: bareos-dir [options]\n" +" -c use as configuration file or directory\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" " -f run in foreground (for debugging)\n" diff --git a/src/stored/bcopy.c b/src/stored/bcopy.c index 4a64dd450fc..0523febec19 100644 --- a/src/stored/bcopy.c +++ b/src/stored/bcopy.c @@ -58,7 +58,7 @@ PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" "Usage: bcopy [-d debug_level] \n" " -b bootstrap specify a bootstrap file\n" -" -c specify a Storage configuration file\n" +" -c specify a Storage configuration file or directory\n" " -D specify a director name specified in the Storage\n" " configuration file for the Key Encryption Key selection\n" " -d set debug level to \n" diff --git a/src/stored/bextract.c b/src/stored/bextract.c index e06dea466e0..270d622bd87 100644 --- a/src/stored/bextract.c +++ b/src/stored/bextract.c @@ -69,7 +69,7 @@ PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" "Usage: bextract \n" " -b specify a bootstrap file\n" -" -c specify a Storage configuration file\n" +" -c specify a Storage configuration file or directory\n" " -D specify a director name specified in the Storage\n" " configuration file for the Key Encryption Key selection\n" " -d set debug level to \n" diff --git a/src/stored/bls.c b/src/stored/bls.c index 67e43dff333..5e47c8df7e5 100644 --- a/src/stored/bls.c +++ b/src/stored/bls.c @@ -62,7 +62,7 @@ PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" "Usage: bls [options] \n" " -b specify a bootstrap file\n" -" -c specify a Storage configuration file\n" +" -c specify a Storage configuration file or directory\n" " -D specify a director name specified in the Storage\n" " configuration file for the Key Encryption Key selection\n" " -d set debug level to \n" diff --git a/src/stored/bscan.c b/src/stored/bscan.c index caf4530b5fa..0dbe443f7fc 100644 --- a/src/stored/bscan.c +++ b/src/stored/bscan.c @@ -104,7 +104,7 @@ PROG_COPYRIGHT "Usage: bscan [ options ] \n" " -B specify the database driver name (default NULL) \n" " -b specify a bootstrap file\n" -" -c specify storage daemon configuration file\n" +" -c specify a Storage configuration file or directory\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" " -m update media info in database\n" diff --git a/src/stored/btape.c b/src/stored/btape.c index b59448429b9..07019dec598 100644 --- a/src/stored/btape.c +++ b/src/stored/btape.c @@ -3009,7 +3009,7 @@ PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" "Usage: btape \n" " -b specify bootstrap file\n" -" -c set Storage configuration file to file\n" +" -c specify Storage configuration file or directory\n" " -D specify a director name specified in the Storage\n" " configuration file for the Key Encryption Key selection\n" " -d set debug level to \n" diff --git a/src/stored/stored.c b/src/stored/stored.c index 25c15f30866..4a3cc5da68c 100644 --- a/src/stored/stored.c +++ b/src/stored/stored.c @@ -68,8 +68,8 @@ static void usage() fprintf(stderr, _( PROG_COPYRIGHT "\nVersion: %s (%s)\n\n" -"Usage: bareos-sd [options] [-c config_file] [config_file]\n" -" -c use as configuration file\n" +"Usage: bareos-sd [options]\n" +" -c use as configuration file or directory\n" " -d set debug level to \n" " -dt print timestamp in debug output\n" " -f run in foreground (for debugging)\n" From d5c4ba494647d2132fe5b8369dc293c054104e0b Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 19 Aug 2016 18:54:45 +0200 Subject: [PATCH 09/15] call validate_resource also from configure add. This is required to check if all required items are set on job resources, because job resources have special treatment during reading the configuration, as they can inherit values from jobdefs. --- src/dird/dird_conf.c | 184 ++++++++++++++++++++++------------------ src/dird/dird_conf.h | 3 + src/dird/ua_configure.c | 20 +++++ src/lib/parse_conf.c | 31 ++++++- src/lib/parse_conf.h | 9 +- 5 files changed, 161 insertions(+), 86 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 1509010c31e..1c9830af8ed 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1256,17 +1256,29 @@ static void propagate_resource(RES_ITEM *items, BRSRES *source, BRSRES *dest) } } + /* * Ensure that all required items are present */ -static bool validate_resource(RES_ITEM *items, BRSRES *res, const char *res_type) +bool validate_resource(int res_type, RES_ITEM *items, BRSRES *res) { + if (res_type == R_JOBDEFS) { + /* + * a jobdef don't have to be fully defined. + */ + return true; + } else if (res_type == R_JOB) { + if (!((JOBRES *)res)->validate()) { + return false; + } + } + for (int i = 0; items[i].name; i++) { if (items[i].flags & CFG_ITEM_REQUIRED) { if (!bit_is_set(i, res->hdr.item_present)) { Jmsg(NULL, M_ERROR, 0, _("\"%s\" directive in %s \"%s\" resource is required, but not found.\n"), - items[i].name, res_type, res->name()); + items[i].name, res_to_str(res_type), res->name()); return false; } } @@ -1275,9 +1287,47 @@ static bool validate_resource(RES_ITEM *items, BRSRES *res, const char *res_type * If this triggers, take a look at lib/parse_conf.h */ if (i >= MAX_RES_ITEMS) { - Emsg1(M_ERROR, 0, _("Too many items in %s resource\n"), res_type); + Emsg1(M_ERROR, 0, _("Too many items in %s resource\n"), res_to_str(res_type)); + return false; + } + } + + return true; +} + +bool JOBRES::validate() +{ + /* + * For Copy and Migrate we can have Jobs without a client or fileset. + * As for a copy we use the original Job as a reference for the Read storage + * we also don't need to check if there is an explicit storage definition in + * either the Job or the Read pool. + */ + switch (JobType) { + case JT_COPY: + case JT_MIGRATE: + break; + default: + /* + * All others must have a client and fileset. + */ + if (!client) { + Jmsg(NULL, M_ERROR, 0, + _("\"client\" directive in Job \"%s\" resource is required, but not found.\n"), name()); + return false; + } + + if (!fileset) { + Jmsg(NULL, M_ERROR, 0, + _("\"fileset\" directive in Job \"%s\" resource is required, but not found.\n"), name()); + return false; + } + + if (!storage && !pool->storage) { + Jmsg(NULL, M_ERROR, 0, _("No storage specified in Job \"%s\" nor in Pool.\n"), name()); return false; } + break; } return true; @@ -2958,7 +3008,7 @@ bool save_resource(int type, RES_ITEM *items, int pass) /* * Ensure that all required items are present */ - if (!validate_resource(items, &res_all.res_dir, resources[rindex].name)) { + if (!validate_resource(type, items, &res_all.res_dir)) { return false; } } @@ -3008,6 +3058,50 @@ bool save_resource(int type, RES_ITEM *items, int pass) return true; } +bool propagate_jobdefs(int res_type, JOBRES *res) +{ + JOBRES *jobdefs = NULL; + + if (!res->jobdefs) { + return true; + } + + /* + * Don't allow the JobDefs pointing to itself. + */ + if (res->jobdefs == res) { + return false; + } + + if (res_type == R_JOB) { + jobdefs = res->jobdefs; + + /* + * Handle RunScripts alists specifically + */ + if (jobdefs->RunScripts) { + RUNSCRIPT *rs, *elt; + + if (!res->RunScripts) { + res->RunScripts = New(alist(10, not_owned_by_alist)); + } + + foreach_alist(rs, jobdefs->RunScripts) { + elt = copy_runscript(rs); + elt->from_jobdef = true; + res->RunScripts->append(elt); /* we have to free it */ + } + } + } + + /* + * Transfer default items from JobDefs Resource + */ + propagate_resource(job_items, res->jobdefs, res); + + return true; +} + /* * Populate Job Defaults (e.g. JobDefs) */ @@ -3020,89 +3114,23 @@ static inline bool populate_jobdefs() * Propagate the content of a JobDefs to another. */ foreach_res(jobdefs, R_JOBDEFS) { - /* - * Don't allow the JobDefs pointing to itself. - */ - if (!jobdefs->jobdefs || jobdefs->jobdefs == jobdefs) { - continue; - } - - propagate_resource(job_items, jobdefs->jobdefs, jobdefs); + propagate_jobdefs(R_JOBDEFS, jobdefs); } /* * Propagate the content of the JobDefs to the actual Job. */ foreach_res(job, R_JOB) { - if (job->jobdefs) { - jobdefs = job->jobdefs; - - /* - * Handle RunScripts alists specifically - */ - if (jobdefs->RunScripts) { - RUNSCRIPT *rs, *elt; - - if (!job->RunScripts) { - job->RunScripts = New(alist(10, not_owned_by_alist)); - } - - foreach_alist(rs, jobdefs->RunScripts) { - elt = copy_runscript(rs); - elt->from_jobdef = true; - job->RunScripts->append(elt); /* we have to free it */ - } - } - - /* - * Transfer default items from JobDefs Resource - */ - propagate_resource(job_items, jobdefs, job); - } + propagate_jobdefs(R_JOB, job); /* * Ensure that all required items are present */ - if (!validate_resource(job_items, job, "Job")) { + if (!validate_resource(R_JOB, job_items, job)) { retval = false; goto bail_out; } - /* - * For Copy and Migrate we can have Jobs without a client or fileset. - * As for a copy we use the original Job as a reference for the Read storage - * we also don't need to check if there is an explicit storage definition in - * either the Job or the Read pool. - */ - switch (job->JobType) { - case JT_COPY: - case JT_MIGRATE: - break; - default: - /* - * All others must have a client and fileset. - */ - if (!job->client) { - Jmsg(NULL, M_ERROR_TERM, 0, - _("\"client\" directive in Job \"%s\" resource is required, but not found.\n"), job->name()); - retval = false; - goto bail_out; - } - - if (!job->fileset) { - Jmsg(NULL, M_ERROR_TERM, 0, - _("\"fileset\" directive in Job \"%s\" resource is required, but not found.\n"), job->name()); - retval = false; - goto bail_out; - } - - if (!job->storage && !job->pool->storage) { - Jmsg(NULL, M_FATAL, 0, _("No storage specified in Job \"%s\" nor in Pool.\n"), job->name()); - retval = false; - goto bail_out; - } - break; - } } /* End loop over Job res */ bail_out: @@ -3111,15 +3139,7 @@ static inline bool populate_jobdefs() bool populate_defs() { - bool retval; - - retval = populate_jobdefs(); - if (!retval) { - goto bail_out; - } - -bail_out: - return retval; + return populate_jobdefs(); } static void store_pooltype(LEX *lc, RES_ITEM *item, int index, int pass) diff --git a/src/dird/dird_conf.h b/src/dird/dird_conf.h index 6f589452ab5..bd59ef0dcc4 100644 --- a/src/dird/dird_conf.h +++ b/src/dird/dird_conf.h @@ -443,6 +443,7 @@ class JOBRES : public BRSRES { runtime_job_status_t *rjs; /* Runtime Job Status */ /* Methods */ + bool validate(); }; #undef MAX_FOPTS @@ -621,6 +622,8 @@ union URES { }; void init_dir_config(CONFIG *config, const char *configfile, int exit_code); +bool propagate_jobdefs(int res_type, JOBRES *res); +bool validate_resource(int type, RES_ITEM *items, BRSRES *res); #define GetPoolResWithName(x) ((POOLRES *)GetResWithName(R_POOL, (x))) #define GetStoreResWithName(x) ((STORERES *)GetResWithName(R_STORAGE, (x))) diff --git a/src/dird/ua_configure.c b/src/dird/ua_configure.c index dfcf3f881b2..3eaf490763d 100644 --- a/src/dird/ua_configure.c +++ b/src/dird/ua_configure.c @@ -288,6 +288,7 @@ static inline bool configure_add_resource(UAContext *ua, int first_parameter, RE POOL_MEM filename_tmp(PM_FNAME); POOL_MEM filename(PM_FNAME); POOL_MEM temp(PM_FNAME); + JOBRES *res = NULL; if (!configure_create_resource_string(ua, first_parameter, res_table, name, resource)) { return false; @@ -316,6 +317,24 @@ static inline bool configure_add_resource(UAContext *ua, int first_parameter, RE return false; } + /* + * parse_config_file has already done some validation. + * However, it skipped at least some checks for R_JOB + * (reason: a job can get values from jobdefs, + * and the value propagation happens after reading the full configuration) + * therefore we explicitly check the new resource here. + */ + if ((res_table->rcode == R_JOB) || (res_table->rcode == R_JOBDEFS)) { + res = (JOBRES *)GetResWithName(res_table->rcode, name.c_str()); + propagate_jobdefs(res_table->rcode, res); + if (!validate_resource(res_table->rcode, res_table->items, (BRSRES *)res)) { + ua->error_msg("failed to create config resource \"%s\"\n", name.c_str()); + unlink(filename_tmp.c_str()); + my_config->remove_resource(res_table->rcode, name.c_str()); + return false; + } + } + /* * new config resource is working fine. Rename file to its permanent name. */ @@ -425,3 +444,4 @@ bool configure_cmd(UAContext *ua, const char *cmd) return result; } + diff --git a/src/lib/parse_conf.c b/src/lib/parse_conf.c index 8652c7b5d28..c9963651ecf 100644 --- a/src/lib/parse_conf.c +++ b/src/lib/parse_conf.c @@ -352,14 +352,41 @@ bool CONFIG::parse_config_file(const char *cf, void *caller_ctx, LEX_ERROR_HANDL return false; } +const char *CONFIG::get_resource_type_name(int code) +{ + return res_to_str(code); +} + +int CONFIG::get_resource_table_index(int resource_type) +{ + int rindex = -1; + + if ((resource_type >= m_r_first) && (resource_type <= m_r_last)) { + rindex = resource_type = m_r_first; + } + + return rindex; +} + +RES_TABLE *CONFIG::get_resource_table(int resource_type) +{ + RES_TABLE *result = NULL; + int rindex = get_resource_table_index(resource_type); + + if (rindex >= 0) { + result = &m_resources[rindex]; + } + + return result; +} -RES_TABLE *CONFIG::get_resource_table(const char *resource_type) +RES_TABLE *CONFIG::get_resource_table(const char *resource_type_name) { RES_TABLE *result = NULL; int i; for (i = 0; m_resources[i].name; i++) { - if (bstrcasecmp(m_resources[i].name, resource_type)) { + if (bstrcasecmp(m_resources[i].name, resource_type_name)) { result = &m_resources[i]; } } diff --git a/src/lib/parse_conf.h b/src/lib/parse_conf.h index e34aaaa5da1..29fb61637da 100644 --- a/src/lib/parse_conf.h +++ b/src/lib/parse_conf.h @@ -329,7 +329,7 @@ class BRSRES { /* Methods */ char *name() const; - bool print_config(POOL_MEM &buf, bool hide_sensitive_data = false); + bool print_config(POOL_MEM &buf, bool hide_sensitive_data = false, bool verbose = false); /* * validate can be defined by inherited classes, * when special rules for this resource type must be checked. @@ -338,6 +338,7 @@ class BRSRES { }; inline char *BRSRES::name() const { return this->hdr.name; } +inline bool BRSRES::validate() { return true; } /* * Message Resource @@ -435,7 +436,10 @@ class CONFIG { bool remove_resource(int type, const char *name); void dump_resources(void sendit(void *sock, const char *fmt, ...), void *sock, bool hide_sensitive_data = false); - RES_TABLE *get_resource_table(const char *resource_type); + const char *get_resource_type_name(int code); + int get_resource_code(const char *resource_type); + RES_TABLE *get_resource_table(int resource_type); + RES_TABLE *get_resource_table(const char *resource_type_name); int get_resource_item_index(RES_ITEM *res_table, const char *item); RES_ITEM *get_resource_item(RES_ITEM *res_table, const char *item); bool get_path_of_resource(POOL_MEM &path, const char *component, const char *resourcetype, @@ -457,6 +461,7 @@ class CONFIG { bool get_config_file(POOL_MEM &full_path, const char *config_dir, const char *config_filename); bool get_config_include_path(POOL_MEM &full_path, const char *config_dir); bool find_config_path(POOL_MEM &full_path); + int get_resource_table_index(int resource_type); }; CONFIG *new_config_parser(); From 693d5ce4444ec57f571b754464b38d6fbc82416e Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 26 Aug 2016 14:57:54 +0200 Subject: [PATCH 10/15] Debian: removed outdated postinst scripts These postinstall scripts tried to set permissions on files that no longer exists. Therefore these scripts get removed. To simplyfy Debian postinst scripts, common operations have been moved into the deploy_config function. --- autoconf/configure.in | 4 -- debian/bareos-director.postinst.in | 11 ++--- debian/bareos-filedaemon.postinst.in | 23 +--------- debian/bareos-storage-ceph.postinst.in | 45 ------------------- debian/bareos-storage-fifo.postinst.in | 45 ------------------- debian/bareos-storage-glusterfs.postinst.in | 45 ------------------- debian/bareos-storage-tape.postinst.in | 45 ------------------- debian/bareos-storage.postinst.in | 8 +--- debian/bareos-traymonitor.postinst.in | 10 ++--- scripts/bareos-config-lib.sh.in | 49 ++++++++++++++++++--- src/console/bconsole.conf.in | 6 +-- 11 files changed, 53 insertions(+), 238 deletions(-) delete mode 100644 debian/bareos-storage-ceph.postinst.in delete mode 100644 debian/bareos-storage-fifo.postinst.in delete mode 100644 debian/bareos-storage-glusterfs.postinst.in delete mode 100644 debian/bareos-storage-tape.postinst.in diff --git a/autoconf/configure.in b/autoconf/configure.in index 6c181b4cf3a..b4f8b0f7bda 100644 --- a/autoconf/configure.in +++ b/autoconf/configure.in @@ -4241,13 +4241,9 @@ debian|ubuntu|univention) debian/bareos-filedaemon.postinst \ debian/bareos-filedaemon.preinst \ debian/bareos-storage-ceph.install \ - debian/bareos-storage-ceph.postinst \ debian/bareos-storage-glusterfs.install \ - debian/bareos-storage-glusterfs.postinst \ - debian/bareos-storage-fifo.postinst \ debian/bareos-storage-fifo.install \ debian/bareos-storage-python-plugin.install \ - debian/bareos-storage-tape.postinst \ debian/bareos-storage-tape.install \ debian/bareos-storage.bareos-sd.init \ debian/bareos-storage.install \ diff --git a/debian/bareos-director.postinst.in b/debian/bareos-director.postinst.in index 4bf39a096ec..cd14910f25b 100644 --- a/debian/bareos-director.postinst.in +++ b/debian/bareos-director.postinst.in @@ -25,18 +25,13 @@ CONFIGDIR="/etc/bareos/bareos-dir.d/" permissions() { - for dir in ${CONFIGDIR} /etc/bareos/bareos-dir-export/; do - chown ${daemon_user}:${daemon_group} ${dir} - chmod 750 ${dir} - done + chown ${daemon_user}:${daemon_group} /etc/bareos/bareos-dir-export/ + chmod 750 /etc/bareos/bareos-dir-export/ } case "$1" in configure) - if ! /usr/lib/bareos/scripts/bareos-config has_config "bareos-dir"; then - /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" - /usr/lib/bareos/scripts/bareos-config initialize_local_hostname - /usr/lib/bareos/scripts/bareos-config initialize_passwords + if /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR"; then /usr/lib/bareos/scripts/bareos-config initialize_database_driver /usr/lib/bareos/scripts/bareos-config apply_dbconfig_settings fi diff --git a/debian/bareos-filedaemon.postinst.in b/debian/bareos-filedaemon.postinst.in index b84fe4e3ed3..9562632653b 100644 --- a/debian/bareos-filedaemon.postinst.in +++ b/debian/bareos-filedaemon.postinst.in @@ -17,33 +17,12 @@ set -e # for details, see http://www.debian.org/doc/debian-policy/ or # the debian-policy package -daemon_user=bareos -daemon_group=bareos - -#director_daemon_user=$daemon_user -#storage_daemon_user=$daemon_user -file_daemon_user=root -#storage_daemon_group=$daemon_group - DEFCONFIGDIR="@configtemplatedir@/bareos-fd.d/" CONFIGDIR="/etc/bareos/bareos-fd.d/" - - -permissions() -{ - chown ${file_daemon_user}:${daemon_group} ${CONFIGDIR} - chmod 750 ${CONFIGDIR} -} - case "$1" in configure) - if ! /usr/lib/bareos/scripts/bareos-config has_config "bareos-fd"; then - /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" - /usr/lib/bareos/scripts/bareos-config initialize_local_hostname - /usr/lib/bareos/scripts/bareos-config initialize_passwords - fi - permissions + /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" || true # on Univention distributions the ucr command allows to open the firewall if [ -x "/etc/init.d/univention-firewall" ] && which ucr >/dev/null 2>&1; then ucr set \ diff --git a/debian/bareos-storage-ceph.postinst.in b/debian/bareos-storage-ceph.postinst.in deleted file mode 100644 index d43868331fb..00000000000 --- a/debian/bareos-storage-ceph.postinst.in +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# postinst script for bareos -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -daemon_user=bareos -daemon_group=bareos - -permissions() -{ - chown root:${daemon_group} /etc/bareos/bareos-sd.d/device-ceph-rados.conf -} - -case "$1" in - configure) - permissions - ;; - abort-upgrade|abort-remove|abort-deconfigure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/bareos-storage-fifo.postinst.in b/debian/bareos-storage-fifo.postinst.in deleted file mode 100644 index 49d9612f7fa..00000000000 --- a/debian/bareos-storage-fifo.postinst.in +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# postinst script for bareos -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -daemon_user=bareos -daemon_group=bareos - -permissions() -{ - chown root:${daemon_group} /etc/bareos/bareos-sd.d/device-fifo.conf -} - -case "$1" in - configure) - permissions - ;; - abort-upgrade|abort-remove|abort-deconfigure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/bareos-storage-glusterfs.postinst.in b/debian/bareos-storage-glusterfs.postinst.in deleted file mode 100644 index 88bfa4b10d4..00000000000 --- a/debian/bareos-storage-glusterfs.postinst.in +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# postinst script for bareos -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -daemon_user=bareos -daemon_group=bareos - -permissions() -{ - chown root:${daemon_group} /etc/bareos/bareos-sd.d/device-gluster.conf -} - -case "$1" in - configure) - permissions - ;; - abort-upgrade|abort-remove|abort-deconfigure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/bareos-storage-tape.postinst.in b/debian/bareos-storage-tape.postinst.in deleted file mode 100644 index a196cd84275..00000000000 --- a/debian/bareos-storage-tape.postinst.in +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# postinst script for bareos -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -daemon_user=bareos -daemon_group=bareos - -permissions() -{ - chown root:${daemon_group} /etc/bareos/bareos-sd.d/device-tape-with-autoloader.conf -} - -case "$1" in - configure) - permissions - ;; - abort-upgrade|abort-remove|abort-deconfigure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/bareos-storage.postinst.in b/debian/bareos-storage.postinst.in index fe6fa924bfd..ef46fac1961 100644 --- a/debian/bareos-storage.postinst.in +++ b/debian/bareos-storage.postinst.in @@ -32,19 +32,13 @@ CONFIGDIR="/etc/bareos/bareos-sd.d/" permissions() { - chown ${storage_daemon_user}:${daemon_group} ${CONFIGDIR} - chmod 750 ${CONFIGDIR} chown ${storage_daemon_user}:${daemon_group} /var/lib/bareos/storage/ chmod 750 /var/lib/bareos/storage/ } case "$1" in configure) - if ! /usr/lib/bareos/scripts/bareos-config has_config "bareos-sd"; then - /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" - /usr/lib/bareos/scripts/bareos-config initialize_local_hostname - /usr/lib/bareos/scripts/bareos-config initialize_passwords - fi + /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" || true permissions /usr/lib/bareos/scripts/bareos-config setup_sd_user # on Univention distributions the ucr command allows to open the firewall diff --git a/debian/bareos-traymonitor.postinst.in b/debian/bareos-traymonitor.postinst.in index 41e8f84594e..35993b2a4d3 100644 --- a/debian/bareos-traymonitor.postinst.in +++ b/debian/bareos-traymonitor.postinst.in @@ -25,17 +25,13 @@ CONFIGDIR="/etc/bareos/tray-monitor.d/" permissions() { - chown root:${daemon_group} ${CONFIGDIR} - chmod 755 ${CONFIGDIR} + chown -R root:${daemon_group} ${CONFIGDIR} + chmod -R 755 ${CONFIGDIR} } case "$1" in configure) - if ! /usr/lib/bareos/scripts/bareos-config has_config "tray-monitor"; then - /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" - /usr/lib/bareos/scripts/bareos-config initialize_local_hostname - /usr/lib/bareos/scripts/bareos-config initialize_passwords - fi + /usr/lib/bareos/scripts/bareos-config deploy_config "$DEFCONFIGDIR" "$CONFIGDIR" || true permissions ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/scripts/bareos-config-lib.sh.in b/scripts/bareos-config-lib.sh.in index 69d6ea95b22..8df0ee2335e 100644 --- a/scripts/bareos-config-lib.sh.in +++ b/scripts/bareos-config-lib.sh.in @@ -15,7 +15,10 @@ STORAGE_DAEMON_USER="@sd_user@" STORAGE_DAEMON_GROUP="@sd_group@" DIRECTOR_DAEMON_USER="@dir_user@" DIRECTOR_DAEMON_GROUP="@dir_group@" +DAEMON_USER="$DIRECTOR_DAEMON_USER" +DAEMON_GROUP="$DIRECTOR_DAEMON_GROUP" +DB_NAME="@DB_NAME@" DB_VERSION="@BDB_VERSION@" SQL_DDL_DIR="@scriptdir@/ddl" SQLITE_BINDIR="@SQLITE_BINDIR@" @@ -107,6 +110,16 @@ get_group_dir() echo "${DIRECTOR_DAEMON_GROUP}" } +get_bareos_user() +{ + echo "${DAEMON_USER}" +} + +get_bareos_dir() +{ + echo "${DAEMON_GROUP}" +} + get_working_dir() { echo "${WORKING_DIR}" @@ -338,15 +351,37 @@ has_config() deploy_config() { + # + # Copy configuration from SOURCE to DEST and apply settings. + # Only copies new files. It does not overwrite existing files. + # + # Returns: + # 0: if new config files have been copied + # 1: otherwise (on error or if no new config files are available) + # + SOURCE=${1:-} DEST=${2:-} + local RC=1 + [ -d "$SOURCE" ] || return 1 [ -d "$DEST" ] || return 1 - # copy all files, but do not overwrite existing file. - cp --recursive --no-clobber "$SOURCE/." "$DEST/." - true + # + # Copy all files, but do not overwrite existing file. + # Use verbose to detect, when new files have been copied. + # + OUT=`cp --verbose --recursive --no-clobber "$SOURCE/." "$DEST/."` + if [ "$OUT" ]; then + initialize_local_hostname + initialize_passwords + chown -R ${DAEMON_USER}:${DAEMON_GROUP} "$DEST" + chmod -R 750 "$DEST" + RC=0 + fi + + return $RC } get_config_param() @@ -824,7 +859,7 @@ apply_dbconfig_settings() # as ${dbc_basepath}/${dbc_dbname}.db # Therefore a link is created. if [ -r "${dbc_basepath}/${dbc_dbname}" ]; then - BAREOS_SQLITE_DB="`get_working_dir`/`get_database_name`.db" + BAREOS_SQLITE_DB="`get_working_dir`/`get_database_name $DB_NAME`.db" if [ ! -f $BAREOS_SQLITE_DB ]; then ln -s "${dbc_basepath}/${dbc_dbname}" "$BAREOS_SQLITE_DB" ls -a `get_working_dir`/bareos* || true @@ -881,10 +916,10 @@ initialize_local_hostname() # Replace all XXX_REPLACE_WITH_LOCAL_HOSTNAME by the local hostname. # hname=`get_local_hostname` - replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-dir" "${hname}-dir" + #replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-dir" "${hname}-dir" replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-fd" "${hname}-fd" - replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-sd" "${hname}-sd" - replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-mon" "${hname}-mon" + #replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-sd" "${hname}-sd" + #replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX-mon" "${hname}-mon" replace "XXX_REPLACE_WITH_LOCAL_HOSTNAME_XXX" "${hname}" } diff --git a/src/console/bconsole.conf.in b/src/console/bconsole.conf.in index ecb6ad00dce..339347dcf78 100644 --- a/src/console/bconsole.conf.in +++ b/src/console/bconsole.conf.in @@ -3,8 +3,8 @@ # Director { - Name = @basename@-dir - DIRport = @dir_port@ - address = @hostname@ + Name = bareos-dir + address = localhost Password = "@dir_password@" + Description = "Bareos Console credentials for local Director" } From 310991b7347ff008f4ec56018ebfd7cff1672adb Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 26 Aug 2016 16:45:50 +0200 Subject: [PATCH 11/15] build: Rebuild configure. --- configure | 4 ---- 1 file changed, 4 deletions(-) diff --git a/configure b/configure index de48bdb748d..00de70d7fb9 100755 --- a/configure +++ b/configure @@ -33435,13 +33435,9 @@ debian|ubuntu|univention) debian/bareos-filedaemon.postinst \ debian/bareos-filedaemon.preinst \ debian/bareos-storage-ceph.install \ - debian/bareos-storage-ceph.postinst \ debian/bareos-storage-glusterfs.install \ - debian/bareos-storage-glusterfs.postinst \ - debian/bareos-storage-fifo.postinst \ debian/bareos-storage-fifo.install \ debian/bareos-storage-python-plugin.install \ - debian/bareos-storage-tape.postinst \ debian/bareos-storage-tape.install \ debian/bareos-storage.bareos-sd.init \ debian/bareos-storage.install \ From 89cf5035f25188657aef00c17e1f9e49c671b3ea Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Fri, 26 Aug 2016 17:41:54 +0200 Subject: [PATCH 12/15] Configuration: add example files Add example Director storage resources for extra storage backends. Debian: improved traymonitor configuration packaging. --- debian/bareos-director.install.in | 2 +- debian/bareos-filedaemon.install.in | 2 +- debian/bareos-storage-ceph.install.in | 1 + debian/bareos-storage-fifo.install.in | 1 + debian/bareos-storage-glusterfs.install.in | 1 + debian/bareos-storage-tape.install.in | 3 +- debian/bareos-storage.install.in | 2 +- platforms/packaging/bareos.spec | 6 ++- src/stored/backends/Makefile.in | 44 ++++++++++--------- .../bareos-dir.d/storage/Gluster.conf.example | 8 ++++ .../bareos-dir.d/storage/Rados.conf.example | 8 ++++ .../bareos-dir.d/storage/NULL.conf.example | 8 ++++ .../bareos-dir.d/storage/Tape.conf.example | 8 ++++ .../autochanger-0.conf.example | 0 .../device/tapedrive-0.conf.example | 2 +- 15 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 src/stored/backends/gfapi_device.d/bareos-dir.d/storage/Gluster.conf.example create mode 100644 src/stored/backends/rados_device.d/bareos-dir.d/storage/Rados.conf.example create mode 100644 src/stored/backends/unix_fifo_device.d/bareos-dir.d/storage/NULL.conf.example create mode 100644 src/stored/backends/unix_tape_device.d/bareos-dir.d/storage/Tape.conf.example rename src/stored/backends/unix_tape_device.d/bareos-sd.d/{device => autochanger}/autochanger-0.conf.example (100%) diff --git a/debian/bareos-director.install.in b/debian/bareos-director.install.in index dac5cde56a1..19511a1b1f2 100644 --- a/debian/bareos-director.install.in +++ b/debian/bareos-director.install.in @@ -26,7 +26,7 @@ @confdir@/bareos-dir.d/schedule/WeeklyCycleAfterBackup.conf @configtemplatedir@/bareos-dir.d/schedule/ @confdir@/bareos-dir.d/schedule/WeeklyCycle.conf @configtemplatedir@/bareos-dir.d/schedule/ @confdir@/bareos-dir.d/storage/File.conf @configtemplatedir@/bareos-dir.d/storage/ -@confdir@/tray-monitor.d/director/Director-local*.conf +@confdir@/tray-monitor.d/director/Director-local*.conf @configtemplatedir@/tray-monitor.d/director/ /usr/sbin/bareos-dir /usr/share/man/man8/bareos-dir.8.gz /usr/share/man/man8/bareos.8.gz diff --git a/debian/bareos-filedaemon.install.in b/debian/bareos-filedaemon.install.in index 7f71b293fe3..dbf768b0a60 100644 --- a/debian/bareos-filedaemon.install.in +++ b/debian/bareos-filedaemon.install.in @@ -2,7 +2,7 @@ @confdir@/bareos-fd.d/director/bareos-dir.conf @configtemplatedir@/bareos-fd.d/director/ @confdir@/bareos-fd.d/director/bareos-mon.conf @configtemplatedir@/bareos-fd.d/director/ @confdir@/bareos-fd.d/messages/Standard.conf @configtemplatedir@/bareos-fd.d/messages/ -@confdir@/tray-monitor.d/client/FileDaemon-local*.conf +@confdir@/tray-monitor.d/client/FileDaemon-local*.conf @configtemplatedir@/tray-monitor.d/client/ /usr/sbin/bareos-fd @plugindir@/bpipe-fd.so /usr/share/man/man8/bareos-fd.8.gz diff --git a/debian/bareos-storage-ceph.install.in b/debian/bareos-storage-ceph.install.in index eaeeb51bcab..5d5b960dedb 100644 --- a/debian/bareos-storage-ceph.install.in +++ b/debian/bareos-storage-ceph.install.in @@ -1,4 +1,5 @@ @backenddir@/libbareossd-rados*.so @backenddir@/libbareossd-cephfs*.so +@confdir@/bareos-dir.d/storage/Rados.conf.example @confdir@/bareos-sd.d/device/RadosStorage.conf.example diff --git a/debian/bareos-storage-fifo.install.in b/debian/bareos-storage-fifo.install.in index 7703ec91625..9241a81631c 100644 --- a/debian/bareos-storage-fifo.install.in +++ b/debian/bareos-storage-fifo.install.in @@ -1,2 +1,3 @@ @backenddir@/libbareossd-fifo*.so +@confdir@/bareos-dir.d/storage/NULL.conf.example @confdir@/bareos-sd.d/device/NULL.conf.example diff --git a/debian/bareos-storage-glusterfs.install.in b/debian/bareos-storage-glusterfs.install.in index e082b3b176f..1f8e63c7cb3 100644 --- a/debian/bareos-storage-glusterfs.install.in +++ b/debian/bareos-storage-glusterfs.install.in @@ -1,2 +1,3 @@ @backenddir@/libbareossd-gfapi*.so +@confdir@/bareos-dir.d/storage/Gluster.conf.example @confdir@/bareos-sd.d/device/GlusterStorage.conf.example diff --git a/debian/bareos-storage-tape.install.in b/debian/bareos-storage-tape.install.in index 700902d4fa7..fbd499c0be1 100644 --- a/debian/bareos-storage-tape.install.in +++ b/debian/bareos-storage-tape.install.in @@ -1,5 +1,6 @@ /etc/bareos/mtx-changer.conf -@confdir@/bareos-sd.d/device/autochanger-0.conf.example +@confdir@/bareos-dir.d/storage/Tape.conf.example +@confdir@/bareos-sd.d/autochanger/autochanger-0.conf.example @confdir@/bareos-sd.d/device/tapedrive-0.conf.example @backenddir@/libbareossd-gentape*.so @backenddir@/libbareossd-tape*.so diff --git a/debian/bareos-storage.install.in b/debian/bareos-storage.install.in index 7bb6428e945..d301af73630 100644 --- a/debian/bareos-storage.install.in +++ b/debian/bareos-storage.install.in @@ -5,6 +5,6 @@ @confdir@/bareos-sd.d/director/bareos-mon.conf @configtemplatedir@/bareos-sd.d/director/ @confdir@/bareos-sd.d/messages/Standard.conf @configtemplatedir@/bareos-sd.d/messages/ @confdir@/bareos-sd.d/storage/bareos-sd.conf @configtemplatedir@/bareos-sd.d/storage/ -@confdir@/tray-monitor.d/storage/StorageDaemon-local*.conf +@confdir@/tray-monitor.d/storage/StorageDaemon-local*.conf @configtemplatedir@/tray-monitor.d/storage/ /usr/sbin/bareos-sd /usr/share/man/man8/bareos-sd.8.gz diff --git a/platforms/packaging/bareos.spec b/platforms/packaging/bareos.spec index 79a2a427440..e9acfb04afe 100644 --- a/platforms/packaging/bareos.spec +++ b/platforms/packaging/bareos.spec @@ -999,7 +999,8 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %{_mandir}/man8/btape.8.gz %{_sbindir}/bscrypto %{_sbindir}/btape -%{_sysconfdir}/bareos/bareos-sd.d/device/autochanger-0.conf.example +%{_sysconfdir}/bareos/bareos-dir.d/storage/Tape.conf.example +%{_sysconfdir}/bareos/bareos-sd.d/autochanger/autochanger-0.conf.example %{_sysconfdir}/bareos/bareos-sd.d/device/tapedrive-0.conf.example %{plugin_dir}/scsicrypto-sd.so %{plugin_dir}/scsitapealert-sd.so @@ -1007,12 +1008,14 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %files storage-fifo %defattr(-, root, root) %{backend_dir}/libbareossd-fifo*.so +%{_sysconfdir}/bareos/bareos-dir.d/storage/NULL.conf.example %{_sysconfdir}/bareos/bareos-sd.d/device/NULL.conf.example %if 0%{?glusterfs} %files storage-glusterfs %defattr(-, root, root) %{backend_dir}/libbareossd-gfapi*.so +%{_sysconfdir}/bareos/bareos-dir.d/storage/Gluster.conf.example %{_sysconfdir}/bareos/bareos-sd.d/device/GlusterStorage.conf.example %endif @@ -1021,6 +1024,7 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %defattr(-, root, root) %{backend_dir}/libbareossd-rados*.so %{backend_dir}/libbareossd-cephfs*.so +%{_sysconfdir}/bareos/bareos-dir.d/storage/Rados.conf.example %{_sysconfdir}/bareos/bareos-sd.d/device/RadosStorage.conf.example %endif diff --git a/src/stored/backends/Makefile.in b/src/stored/backends/Makefile.in index f7e15d74980..139df7fef24 100644 --- a/src/stored/backends/Makefile.in +++ b/src/stored/backends/Makefile.in @@ -177,28 +177,30 @@ libtool-install: all install-config: # copy configuration resource files to directory structure - @SRCCONFDIR=$(CONF_EXTRA_DIR)/bareos-sd.d/; \ - DESTCONFDIR=${DESTDIR}${confdir}/bareos-sd.d/; \ - echo "installing configuration resource files to $$DESTCONFDIR"; \ - $(MKDIR) $$DESTCONFDIR; \ - for RESTYPE in autochanger device director ndmp messages storage; do \ - $(MKDIR) "$$DESTCONFDIR/$$RESTYPE"; \ - for f in $$SRCCONFDIR/$$RESTYPE/*.conf $$SRCCONFDIR/$$RESTYPE/*.conf.example; do \ - if test -e "$$f"; then \ - RESDIR="$$DESTCONFDIR/$$RESTYPE"; \ - RES=`basename "$$f"`; \ - if test -f "$$RESDIR/$$RES"; then \ - destconf="$$RESDIR/$$RES.new"; \ - echo " $$RESTYPE/$$RES as \"$$destconf\" (keep existing)"; \ - else \ - destconf="$$RESDIR/$$RES"; \ - echo " $$RESTYPE/$$RES as \"$$destconf\""; \ + @for COMPONENT in bareos-dir.d bareos-sd.d; do \ + SRCCONFDIR=$(CONF_EXTRA_DIR)/$$COMPONENT/; \ + DESTCONFDIR=${DESTDIR}${confdir}/$$COMPONENT/; \ + echo "installing configuration resource files to $$DESTCONFDIR"; \ + $(MKDIR) $$DESTCONFDIR; \ + for RESTYPE in autochanger device director ndmp messages storage; do \ + for f in $$SRCCONFDIR/$$RESTYPE/*.conf $$SRCCONFDIR/$$RESTYPE/*.conf.example; do \ + if test -e "$$f"; then \ + RESDIR="$$DESTCONFDIR/$$RESTYPE"; \ + $(MKDIR) "$$RESDIR"; \ + RES=`basename "$$f"`; \ + if test -f "$$RESDIR/$$RES"; then \ + destconf="$$RESDIR/$$RES.new"; \ + echo " $$RESTYPE/$$RES as \"$$destconf\" (keep existing)"; \ + else \ + destconf="$$RESDIR/$$RES"; \ + echo " $$RESTYPE/$$RES as \"$$destconf\""; \ + fi; \ + ${INSTALL_CONFIG} "$$f" "$$destconf"; \ + if test "x${sd_group}" != "x"; then \ + chgrp -f ${sd_group} $$destconf || true; \ + fi; \ fi; \ - ${INSTALL_CONFIG} "$$f" "$$destconf"; \ - if test "x${sd_group}" != "x"; then \ - chgrp -f ${sd_group} $$destconf || true; \ - fi; \ - fi; \ + done; \ done; \ done diff --git a/src/stored/backends/gfapi_device.d/bareos-dir.d/storage/Gluster.conf.example b/src/stored/backends/gfapi_device.d/bareos-dir.d/storage/Gluster.conf.example new file mode 100644 index 00000000000..e566baf4670 --- /dev/null +++ b/src/stored/backends/gfapi_device.d/bareos-dir.d/storage/Gluster.conf.example @@ -0,0 +1,8 @@ +Storage { + Name = Gluster + Address = "Replace this by the Bareos Storage Daemon FQDN or IP address" + Password = "Replace this by the Bareos Storage Daemon director password" + Device = GlusterStorage + Media Type = GlusterFile +} + diff --git a/src/stored/backends/rados_device.d/bareos-dir.d/storage/Rados.conf.example b/src/stored/backends/rados_device.d/bareos-dir.d/storage/Rados.conf.example new file mode 100644 index 00000000000..9aa848f83bb --- /dev/null +++ b/src/stored/backends/rados_device.d/bareos-dir.d/storage/Rados.conf.example @@ -0,0 +1,8 @@ +Storage { + Name = Rados + Address = "Replace this by the Bareos Storage Daemon FQDN or IP address" + Password = "Replace this by the Bareos Storage Daemon director password" + Device = RadosStorage + Media Type = RadosFile +} + diff --git a/src/stored/backends/unix_fifo_device.d/bareos-dir.d/storage/NULL.conf.example b/src/stored/backends/unix_fifo_device.d/bareos-dir.d/storage/NULL.conf.example new file mode 100644 index 00000000000..a5240c554cf --- /dev/null +++ b/src/stored/backends/unix_fifo_device.d/bareos-dir.d/storage/NULL.conf.example @@ -0,0 +1,8 @@ +Storage { + Name = NULL + Address = "Replace this by the Bareos Storage Daemon FQDN or IP address" + Password = "Replace this by the Bareos Storage Daemon director password" + Device = NULL + Media Type = null +} + diff --git a/src/stored/backends/unix_tape_device.d/bareos-dir.d/storage/Tape.conf.example b/src/stored/backends/unix_tape_device.d/bareos-dir.d/storage/Tape.conf.example new file mode 100644 index 00000000000..460daa3cc31 --- /dev/null +++ b/src/stored/backends/unix_tape_device.d/bareos-dir.d/storage/Tape.conf.example @@ -0,0 +1,8 @@ +Storage { + Name = Tape + Address = "Replace this by the Bareos Storage Daemon FQDN or IP address" + Password = "Replace this by the Bareos Storage Daemon director password" + Device = autochanger-0 + Media Type = LTO +} + diff --git a/src/stored/backends/unix_tape_device.d/bareos-sd.d/device/autochanger-0.conf.example b/src/stored/backends/unix_tape_device.d/bareos-sd.d/autochanger/autochanger-0.conf.example similarity index 100% rename from src/stored/backends/unix_tape_device.d/bareos-sd.d/device/autochanger-0.conf.example rename to src/stored/backends/unix_tape_device.d/bareos-sd.d/autochanger/autochanger-0.conf.example diff --git a/src/stored/backends/unix_tape_device.d/bareos-sd.d/device/tapedrive-0.conf.example b/src/stored/backends/unix_tape_device.d/bareos-sd.d/device/tapedrive-0.conf.example index 70c366e207f..f1920f7a05f 100644 --- a/src/stored/backends/unix_tape_device.d/bareos-sd.d/device/tapedrive-0.conf.example +++ b/src/stored/backends/unix_tape_device.d/bareos-sd.d/device/tapedrive-0.conf.example @@ -24,7 +24,7 @@ Device { # arbitrary string that descripes the the storage media. # Bareos uses this to determine, which device can be handle what media. - MediaType = lto + MediaType = LTO # enable "Check Labels" if tapes with ANSI/IBM labels # should be preserved From f2cfcd20977d13e62de0f4972d6ca0081e1a8077 Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Mon, 29 Aug 2016 10:41:30 +0200 Subject: [PATCH 13/15] build: Use STORED_RESTYPES define for resource types. --- src/stored/backends/Makefile.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stored/backends/Makefile.in b/src/stored/backends/Makefile.in index 139df7fef24..c7e80443dc3 100644 --- a/src/stored/backends/Makefile.in +++ b/src/stored/backends/Makefile.in @@ -61,6 +61,8 @@ BUILD_BACKENDS = @BUILD_SD_BACKENDS@ CONF_EXTRA_DIR = config.d +STORED_RESTYPES = autochanger device director ndmp messages storage + .SUFFIXES: .c .o .lo .PHONY: .DONTCARE: @@ -182,7 +184,7 @@ install-config: DESTCONFDIR=${DESTDIR}${confdir}/$$COMPONENT/; \ echo "installing configuration resource files to $$DESTCONFDIR"; \ $(MKDIR) $$DESTCONFDIR; \ - for RESTYPE in autochanger device director ndmp messages storage; do \ + for RESTYPE in $(STORED_RESTYPES); do \ for f in $$SRCCONFDIR/$$RESTYPE/*.conf $$SRCCONFDIR/$$RESTYPE/*.conf.example; do \ if test -e "$$f"; then \ RESDIR="$$DESTCONFDIR/$$RESTYPE"; \ From 6aae37e971e3195163851ad7d4aafb6f2a427c74 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Mon, 22 Aug 2016 16:12:31 +0200 Subject: [PATCH 14/15] bcommand: Fix status scheduler This prevents displaying disabled schedules in the preview of the status scheduler command. --- src/dird/ua_status.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dird/ua_status.c b/src/dird/ua_status.c index 048ff3392d4..d6894ef3a01 100644 --- a/src/dird/ua_status.c +++ b/src/dird/ua_status.c @@ -765,6 +765,10 @@ static void do_scheduler_status(UAContext *ua) */ LockRes(); foreach_res(sched, R_SCHEDULE) { + if (!schedulegiven && !sched->enabled) { + continue; + } + if (!acl_access_ok(ua, Schedule_ACL, sched->hdr.name)) { continue; } From 85b63f2598e411afaea43b046b62f301a895ec76 Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Mon, 22 Aug 2016 19:06:23 +0200 Subject: [PATCH 15/15] bcommand: Check arguments to .authorized cmd. --- src/dird/ua_dotcmds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dird/ua_dotcmds.c b/src/dird/ua_dotcmds.c index bfe3e2721e4..2bf4daab98f 100644 --- a/src/dird/ua_dotcmds.c +++ b/src/dird/ua_dotcmds.c @@ -76,7 +76,7 @@ bool dot_authorized_cmd(UAContext *ua, const char *cmd) for (int i = 1; i < ua->argc; i++) { for (int j = 0; authorization_mappings[j].type; j++) { if (bstrcasecmp(ua->argk[i], authorization_mappings[j].type)) { - if (acl_access_ok(ua, authorization_mappings[j].acl_type, ua->argv[i], false)) { + if (ua->argv[i] && acl_access_ok(ua, authorization_mappings[j].acl_type, ua->argv[i], false)) { retval = true; } else { retval = false;