diff --git a/CHANGES b/CHANGES index 8258e7d636..015f36e16f 100644 --- a/CHANGES +++ b/CHANGES @@ -43,7 +43,7 @@ btrfs-progs-5.17 (2022-04-26) profiles * preparatory work for extent tree v2, global roots * experimental feature (unstable interface, not built by default, - do not use for production): + do not use for production) * btrfstune: option --csum to switch checksum algorithm * other: * cleanups, refactoring @@ -1190,6 +1190,7 @@ btrfs-progs-4.4 (2016-01-18) * usage=min..max -- enhanced to take range * stripes=min..max -- new, filter by stripes for raid0/10/5/6 * limit=min..max -- enhanced to take range + Note: due to backward compatibility, the range maximum for 'usage' is not inclusive as for the others, to keep the same behaviour as usage=N . diff --git a/Documentation/Balance.rst b/Documentation/Balance.rst index 64f3d7896a..4ea2b4f1db 100644 --- a/Documentation/Balance.rst +++ b/Documentation/Balance.rst @@ -7,3 +7,9 @@ Filters ------- .. include:: ch-balance-filters.rst + + +Filters +------- + +.. include:: ch-balance-examples.rst diff --git a/Documentation/CHANGES.rst b/Documentation/CHANGES.rst index 91d9436303..65e25483d5 100644 --- a/Documentation/CHANGES.rst +++ b/Documentation/CHANGES.rst @@ -1,5 +1,5 @@ -Changes -======= +Changes (btrfs-progs) +===================== Signed release tarballs can be found at https://www.kernel.org/pub/linux/kernel/people/kdave/btrfs-progs/ diff --git a/Documentation/Contributors.rst b/Documentation/Contributors.rst new file mode 100644 index 0000000000..613449fcb0 --- /dev/null +++ b/Documentation/Contributors.rst @@ -0,0 +1,134 @@ +Contributors +============ + +The following companies contribute to Btrfs code, not counting the treewide and +other subsystem changes. Infrequent contributions are not reflected in this +list, please have a look to the git history for complete list. + +Sorted by amount of contributions: + +* SUSE +* Facebook +* Western Digital +* Oracle + +The following contributed in the past (sorted alphabetically): + +* Fujitsu +* Fusion-IO +* Intel +* Linux Foundation +* Red Hat +* STRATO AG + +Statistics for 5.x series +------------------------- + +.. csv-table:: + :header: "Version", "Contributors", "SLOC", "Raw lines", "Patches", "Diffstat" + :align: left + + "5.0", "15", "98298", "134159", "144", "+3173 -2297" + "5.1", "18", "98992", "135308", "116", "+2208 -1059" + "5.2", "22", "99888", "136521", "255", "+3524 -2311" + "5.3", "20", "100254", "137224", "124", "+4106 -3400" + "5.4", "18", "100660", "137889", "166", "+10752 -10087" + "5.5", "17", "100638", "138212", "168", "+3055 -2729" + "5.6", "18", "101482", "139742", "114", "+3370 -1840" + "5.7", "22", "101661", "140694", "221", "+4484 -3532" + "5.8", "21", "101562", "140930", "158", "+3176 -2940" + "5.9", "18", "101973", "141748", "188", "+2218 -1400" + "5.10", "22", "102378", "142760", "187", "+3148 -2135" + "5.11", "15", "102418", "143124", "229", "+4872 -4507" + "5.12", "20", "105026", "147099", "195", "+5310 -1316" + "5.13", "19", "105820", "148503", "145", "+3334 -1930" + "5.14", "19", "106324", "149550", "121", "+2823 -1774" + "5.15", "17", "106895", "151006", "133", "+2879 -1422" + "5.16", "24", "107854", "152760", "173", "+4770 -3016" + "5.17", "17", "107910", "153407", "154", "+4024 -3378" + "5.18", "30", "109159", "155372", "143", "+3489 -1523" + +Legend: + +- *Files:* fs/btrfs/\*.[ch], fs/btrfs/tests/\*.[ch], include/uapi/linux/btrfs.h, include/uapi/linux/btrfs_tree.h, include/linux/btrfs.h, include/trace/events/btrfs.h +- *Version:* mainline version +- *Contributors:* number of people that sent patches that modified ''Files'', direct btrfs development or originating from other tree-wide changes +- *SLOC:* lines of code, http://dwheeler.com/sloccount/ (generated by version 2.26) +- *Raw lines:* counted by ''wc -l'' over ''Files'' +- *Patches:* number of patches from ''Contributors'', merge commits excluded +- *Diffstat:* lines added and deleted in ''Files'' + +Statistics for 4.x series +-------------------------- + +.. csv-table:: + :header: "Version", "Contributors", "SLOC", "Raw lines", "Patches", "Diffstat" + :align: left + + "4.0", "22", "85849", "115716", "97", "+1622 -937" + "4.1", "25", "87596", "118253", "120", "+2415 -1062" + "4.2", "19", "87935", "118790", "119", "+2392 -1855" + "4.3", "23", "88384", "119576", "74", "+1516 -730" + "4.4", "26", "89543", "121456", "138", "+3184 -1304" + "4.5", "26", "91708", "124363", "127", "+4370 -1462" + "4.6", "29", "92134", "125045", "100", "+1890 -1208" + "4.7", "33", "92922", "126264", "161", "+3721 -2502" + "4.8", "22", "93769", "127392", "114", "+2732 -1604" + "4.9", "25", "94237", "128040", "64", "+1959 -1311" + "4.10", "24", "94303", "128156", "105", "+4874 -4758" + "4.11", "24", "94365", "128340", "210", "+2084 -1900" + "4.12", "21", "94931", "129230", "85", "+1803 -913" + "4.13", "29", "93892", "127970", "135", "+2017 -1607" + "4.14", "28", "94296", "124346", "132", "+2114 -1520" + "4.15", "30", "97091", "132221", "128", "+3761 -1795" + "4.16", "25", "97637", "133305", "188", "+2562 -1481" + "4.17", "21", "98027", "133003", "194", "+2723 -3024" + "4.18", "26", "98387", "133667", "200", "+3643 -2979" + "4.19", "25", "97547", "132655", "193", "+2058 -3070" + "4.20", "22", "97830", "133283", "128", "+1560 -932" + +Statistics for 3.x series +------------------------- + +.. csv-table:: + :header: "Version", "Contributors", "SLOC", "Raw lines", "Patches", "Diffstat" + :align: left + + "3.0", "25", "48665", "65192", "126", "+7508 -5175" + "3.1", "24", "48647", "65248", "106", "+1762 -1586" + "3.2", "30", "51574", "69552", "184", "+6344 -2040" + "3.3", "27", "56216", "75485", "129", "+7151 -1218" + "3.4", "25", "57865", "77671", "118", "+4597 -2411" + "3.5", "21", "59683", "79983", "108", "+3570 -1258" + "3.6", "25", "65894", "88123", "104", "+9145 -1005" + "3.7", "30", "67348", "90171", "151", "+3802 -1754" + "3.8", "25", "70289", "93916", "161", "+5599 -1854" + "3.9", "29", "73414", "98602", "160", "+6430 -1242" + "3.10", "24", "74449", "99980", "133", "+3529 -2151" + "3.11", "21", "74875", "100657", "100", "+2538 -1857" + "3.12", "32", "76265", "102497", "158", "+4373 -2533" + "3.13", "24", "77532", "104108", "123", "+2741 -1123" + "3.14", "28", "79879", "107069", "171", "+5290 -2329" + "3.15", "27", "80308", "107544", "152", "+2389 -1914" + "3.16", "29", "82292", "110331", "137", "+4361 -1574" + "3.17", "19", "82625", "110841", "44", "+1060 -550" + "3.18", "25", "83910", "112906", "149", "+3696 -1631" + "3.19", "18", "85420", "115031", "82", "+2802 -677" + +Statistics for 2.6.x series +--------------------------- + +.. csv-table:: + :header: "Version", "Contributors", "SLOC", "Raw lines", "Patches", "Diffstat" + :align: left + + "2.6.30", "22", "33838", "45377", "70", "+4403 -2632" + "2.6.31", "19", "38825", "51693", "68", "+9207 -2862" + "2.6.32", "15", "40211", "53515", "95", "+4291 -2469" + "2.6.33", "17", "40408", "53806", "43", "+1332 -1041" + "2.6.34", "18", "41100", "54715", "54", "+1374 -465" + "2.6.35", "14", "43014", "57082", "50", "+5230 -2863" + "2.6.36", "4", "43016", "57088", "6", "+39 -33" + "2.6.37", "17", "44781", "59491", "83", "+3104 -701" + "2.6.38", "23", "46573", "61980", "90", "+3472 -983" + "2.6.39", "28", "47206", "62859", "102", "+2115 -1236" diff --git a/Documentation/DocConventions.rst b/Documentation/DocConventions.rst index a9a87012dd..a3996afdba 100644 --- a/Documentation/DocConventions.rst +++ b/Documentation/DocConventions.rst @@ -28,7 +28,7 @@ Quotation in subcommands: - argument name in option description: caps in angle brackets - reference in help text: caps NAME - also possible: caps italics *NAME* + - also possible: caps italics *NAME* - command short description: - command name: bold **command** diff --git a/Documentation/Feature-by-version.rst b/Documentation/Feature-by-version.rst new file mode 100644 index 0000000000..43d9ca5f80 --- /dev/null +++ b/Documentation/Feature-by-version.rst @@ -0,0 +1,411 @@ +Changes (feature/version) +========================= + +Major features or significant feature enhancements by kernel version. For more +information look below. + +The version states at which version a feature has been merged into the mainline +kernel. It does not tell anything about at which kernel version it is +considered mature enough for production use. For an estimation on stability of +features see [[Status]] page. + +3.x +--- + +3.0 - scrub + Read all data and verify checksums, repair if possible. + +3.2 - auto raid repair + Automatic repair of broken data from a good copy + +3.2 - root backups + Save a few previous versions of the most imporant tree roots at commit time, used by *-o recovery* + +3.3 - integrity checker + Optional infrastructure to verify integrity of written metadata blocks + +3.3 - backref walking + Groundwork to allow tracking owner of blocks, used via *inspect-internal* + +3.3 - restriper + RAID profiles can be changed on-line, balance filters + +3.4 - big metadata blocks + Support for metadata blocks larger than page size + + .. note:: + Default nodesize is 16k since btrfs-progs 3.12 + +3.4 - error handling + Generic infrastructure for graceful error handling (EIO) + +3.5 - device statistics + Persistent statistics about device errors + +3.5 - fsync speedup + Noticeable improvements in fsync() implementation + +3.6 - qgroups + Subvolume-aware quotas + +3.6 - send/receive + Ability to transfer one filesystem via a data stream (full or + incremental) and apply the changes on a remote filesystem. +3.7 - extrefs + Hardlink count limit is lifted to 64k + + .. note:: + Default since btrfs-progs 3.12 + +3.7 - hole puching + Implement the FALLOC_FL_PUNCH_HOLE mode of *fallocate* + +3.8 - device replace + Efficient replacement of existing device (add/remove in one go) + +3.9 - raid 5/6 *(incomplete)* + Basic support for RAD5/6 profiles, no crash resiliency, replace and scrub support + +3.9 - snapshot-aware defrag + Defrag does not break links between shared extents (snapshots, reflinked files) + + .. note:: + Disabled since 3.14 (and backported to some stable kernel versions) + due to problems. Will be enabled in the future. + +3.9 - lightweight send + A mode of *send* that does not add the actual file data to the stream + +3.9 - on-line label set/get + Label editable on mounted filesystems + +3.10 - skinny metadata + Reduced metadata size (format change) of extents + + .. note:: + Default since btrfs-progs 3.18 + +3.10 - qgroup rescan + Sync qgroups with existing filesystem data + +3.12 - uuid tree + A map of subvolume/UUID that vastly speeds up send/receive + +3.12 - out-of-bound dedup + Support for deduplicating extents on a given set of files. + +3.14 - no-holes + No extent representation for file holes (format change), may reduce overall metadata consumption + +3.14 - feature bits in sysfs + /sys/fs/btrfs exports various bits about filesystem capabilities and feature support + +3.16 - O_TMPFILE + Mode of open() to safely create a temporary file + +3.16 - search ioctl v2 + The extended SEARCH_TREE ioctl able to get more than a 4k data + +3.18 - auto blockgroup reclaim + Automatically remove blockgroups (aka. chunks) that become completely empty. + +3.19 - raid56: scrub, replace + Scrub and device replace works on RAID56 filesystems. + +4.x +--- + +4.0 - store otime + Save creation time (otime) for all new files and directories. For + future use, current tool cannot read it directly. + +4.2 - rootid ioctl accessible + The INO_LOOKUP will return root id (id of the containing subvolume), + unrestricted and to all users if the *treeid* is 0. + +4.2 - dedupe possible on the same inode + The EXTENT_SAME ioctl will accept the same inode as source and + destination (ranges must not overlap). + +4.3 - trim all free space + Trim will be performed also on the space that's not allocated by the + chunks, not only free space within the allocated chunks. + +4.4 - balance filter updates + Enhanced syntax and new balance filters: + * limit=min..max + * usage=min..max + * stripes=min..max + +4.5 - free space tree + Improved implementation of free space cache (aka v2), using b-trees. + + .. note:: + Default since btrfs-progs 5.15, Kernel 4.9 fixes endianity bugs on + big-endian machines, x86* is ok + +4.5 - balance filter updates + Conversion to data/DUP profile possible through balance filters -- on single-device filesytem. + + .. note:: + mkfs.btrfs allows creating DUP on single device in the non-mixed mode since 4.4 + +4.6 - max_inline default + The default value of max_inline changed to 2048. + +4.6 - read features from control device + The existing ioctl GET_SUPPORTED_FEATURES can be now used on the + control device (/dev/btrfs-control) and returns the supported features + without any mounted filesystem. + +4.7 - delete device by id + Add new ioctl RM_DEV_V2, pass device to be deleted by its ID. + +4.7 - more renameat2 modes + Add support for RENAME_EXCHANGE and RENAME_WHITEOUT to *renameat2* + syscall. This also means that *overlayfs* is now supported on top of + btrfs. + +4.7 - balance filter updates + Conversion to data/DUP profile possible through balance filters -- on multiple-device filesystems. + + .. note:: + mkfs.btrfs allows creating DUP on multiple devices since 4.5.1 + +4.12 - raid56: auto repair + Scrub will attempt auto-repair (similar to raid1/raid10) + +4.13 - statx + Support for the enhanced statx syscall; file creation timestamp + +4.13 - sysfs qgroups override + qgroups: new sysfs control file to allow temporary quota override with CAP_SYS_RESOURCE + +4.13 - *deprecated mount option alloc_start* + That was a debugging helper, not used and not supposed to be used nowadays. + +4.14 - ZSTD compression + New compression algorithm ZSTD, supposedly better ratio/speed performance. + +4.14 - improved degraded mount + Allow degraded mount based on the chunk constraints, not device number + constraints. Eg. when one device is missing but the remaining one holds + all *single* chunks. + +4.14 - *deprecated user transaction ioctl* + BTRFS_IOC_TRANS_START and BTRFS_IOC_TRANS_END, no known users, tricky + to use; scheduled to be removed in 4.17 + +4.14 - refine SSD optimizations + The mount option *ssd* does not make any assumptions about block layout + or management by the device anymore, leaving only the speedups based on + low seek cost active. This could avoid some corner cases leading to + excessive fragmentation. + https://git.kernel.org/linus/583b723151794e2ff1691f1510b4e43710293875 + The story so far. + +4.15 - overlayfs + Overlayfs can now use btrfs as the lower filesystem. + +4.15 - *ref-verify* + Debugging functionality to verify extent references. New mount option + ref-verify, must be built with CONFIG_BTRFS_FS_REF_VERIFY. + +4.15 - zlib level + Allow to set the zlib compression level via mount option, eg. like + *compress=zlib:9*. The levels match the default zlib compression + levels. The default is 3. + +4.15 - v2 of LOGICAL_INO ioctl + An enhanced version of ioctl that can translate logical extent offset + to inode numbers, "who owns this block". For certain usecases the V1 + performs bad and this is addressed by V2. + [https://git.kernel.org/linus/d24a67b2d997c860a42516076f3315c2ad2d2884 + Read more.] + +4.15 - compression heuristics + Apply a few heuristics to the data before they're compressed to decide + if it's likely to gain any space savings. The methods: frequency + sampling, repeated pattern detection, Shannon entropy calculation. + +4.16 - fallocate: zero range + Mode of the [http://man7.org/linux/man-pages/man2/fallocate.2.html + *fallocate*] syscall to zero file range. + +4.17 - *removed user transaction ioctl* + deprecated in 4.14, see above + +4.17 - *rmdir* on subvolumes + Allow rmdir to delete an empty subvolume. + +4.18 - XFLAGS ioctl + Add support for ioctl FS_IOC_FSSETXATTR/FS_IOC_FSGETXATTR, successor of + FS_IOC_SETFLAGS/FS_IOC_GETFLAGS ioctl. Currently supports: APPEND, + IMMUTABLE, NOATIME, NODUMP, SYNC. Note that the naming is very + confusing, though it's named *xattr*, it does not mean the extended + attributes. It should be referenced as extended inode flags or + *xflags*. + +4.18 - EXTENT_SAME ioctl / 16MiB chunks + The range for out-of-band deduplication implemented by the EXTENT_SAME + ioctl will split the range into 16MiB chunks. Up to now this was the + overall limit and effectively only the first 16MiB was deduplicated. + +4.18 - GET_SUBVOL_INFO ioctl + New ioctl to read subvolume information (id, directory name, + generation, flags, UUIDs, time). This does not require root + permissions, only the regular access to to the subvolume. + +4.18 - GET_SUBVOL_ROOTREF ioctl + New ioctl to enumerate subvolume references of a given subvolume. This + does not require root permissions, only the regular access to to the + subvolume. + +4.18 - INO_LOOKUP_USER ioctl + New ioctl to lookup path by inode number. This does not require root + permissions, only the regular access to to the subvolume, unlike the + INO_LOOKUP ioctl. + +4.19 - defrag ro/rw + Allow to run defrag on files that are normally accesible for + read-write, but are currently opened in read-only mode. + +5.x +--- + +5.0 - swapfile + With some limitations where COW design does not work well with the swap + implementation (nodatacow file, no compression, cannot be snapshotted, + not possible on multiple devices, ...), as this is the most restricted + but working setup, we'll try to improve that in the future + +5.0 - metadata uuid + An optional incompat feature to assign a new filesystem UUID without + overwriting all metadata blocks, stored only in superblock, unlike what + ``btrfstune -u`` + +5.1 - FORGET_DEV ioctl + Unregister devices previously added by the scan ioctl, same effect as + if the kernel module is reloaded. + +5.1 - zstd level + Allow to set the zstd compression level via mount option, eg. like + *compress=zstd:9*. The levels match the default zstd compression + levels. The default is 3, maximum is 15. + +5.2 - pre-write checks + Verify metadata blocks before submitting them to the devices. This can + catch consistency problems or bitflips. + +5.5 - more checksums + New checksum algorithms: xxhash (64b), SHA256 (256b), BLAKE2b (256b). + +5.5 - RAID1C34 + RAID1 with 3- and 4- copies (over all devices). + +5.6 - async discard + Mode of discard (*mount -o discard=async*) that merges freed extents to + larger chunks and submits them for discard in a less intrusive way + +5.6 - device info in sysfs + More information about device state can be found in per-filesystem sysfs directory. + +5.7 - reflink/clone works on inline files + Inline files can be reflinked to the tail extent of other files + +5.7 - faster balance cancel + More cancellation points in balance that will shorten the time to stop + processing once btrfs balance cancel is called. + +5.7 - *removed flag BTRFS_SUBVOL_CREATE_ASYNC* + Remove support of flag BTRFS_SUBVOL_CREATE_ASYNC from subvolume creation ioctl. + +5.7 - v2 of snapshot deletion ioctl + New ioctl BTRFS_IOC_SNAP_DESTROY_V2, deletion by subvolume id is now possible. + +5.9 - mount option *rescue* + Unified mount option for actions that may help to access a damaged + filesystem. Now supports: nologreplay, usebackuproot + +5.9 - qgroups in sysfs + The information about qgroup status and relations is exported in */sys/fs/UUID/qgroups* + +5.9 - FS_INFO ioctl + Export more information: checksum type, checksum size, generation, metadata_uuid + +5.10 - exclusive ops in sysfs + Export which filesystem exclusive operation is running (balance, + resize, device add/delete/relpace, ...) + +5.11 - remove *inode_cache* + Remove inode number caching feature (mount -o inode_cache) + +5.11 - more rescue= + Additional modes for mount option *rescue=*: ignorebadroots/ibadroots, + ignoredatacsums/idatacsums. All are exported in sysfs. + +5.12 - zoned mode + Support for zoned devices with special allocation/write mode to + fixed-size zones. See [[Zoned]]. + +5.13 - supported_sectorsizes in sysfs + List supported sector sizes in sysfs file /sys/fs/btrfs/features/supported_sectorsizes + +5.14 - sysfs scrub bw limit + Tunable bandwidth limit + (/sys/fs/btrfs/FSID/devinfo/DEVID/scrub_speed_max) for scrub (and + device replace) for a given device. + +5.14 - sysfs device stats + The device stats can be also found in /sys/fs/btrfs/FSID/devinfo/DEVID/error_stats. + +5.14 - cancellable resize, device delete + The filesystem resize and device delete operations can be cancelled by + specifying *cancel* as the device name. + +5.14 - property value reset + Change how empty value is interpreted. New behaviour will delete the + value and reset it to default. This affects *btrfs.compression* where + value *no* sets NOCOMPRESS bit while empty value resets all compression + settings (either compression or NOCOMPRESS bit). + +5.15 - fsverity + The fs-verity is a support layer that filesystems can hook into to + support transparent integrity and authenticity protection of read-only + files. https://www.kernel.org/doc/html/latest/filesystems/fsverity.html + +5.15 - idmapped mount + Support mount with uid/gid mapped according to another namespace. + https://lwn.net/Articles/837566/ + +5.16 - ZNS in zoned + Zoned namespaces. https://zonedstorage.io/docs/introduction/zns , + https://lwn.net/Articles/865988/ + +5.17 - send and relocation + Send and relocation (balance, device remove, shrink, block group + reclaim) can now work in parallel + +5.17 - device add vs balance + It is possible to add a device with paused balance + + .. note:: + Since kernel 5.17.7 and btrfs-progs 5.17.1 + +5.17 - *no warning with flushoncommit* + Mounting with *-o flushoncommit* does not triggher the (harmless) + warning at each transaction commit + + .. note:: + Also backported to 5.15.27 and 5.16.13 + +5.18 - zoned and DUP metadata + DUP metadata works with zoned mode + +5.18 - encoded data ioctl + New ioctls to read and write pre-encoded data (ie. no transformation + and directly written as extents), now works for compressed data + +5.18 - *removed balance ioctl v1* + The support for ioctl BTRFS_IOC_BALANCE has been removed, superseded by + BTRFS_IOC_BALANCE_V2m long time ago diff --git a/Documentation/Source-repositories.rst b/Documentation/Source-repositories.rst index cfed2e4c99..2549948012 100644 --- a/Documentation/Source-repositories.rst +++ b/Documentation/Source-repositories.rst @@ -40,17 +40,16 @@ Official repositories The sources of the userspace utilities can be obtained from these repositories: -* git://git.kernel.org/pub/scm/linux/kernel/git/kdave/btrfs-progs.git (`gitweb - access - `_) +* git://git.kernel.org/pub/scm/linux/kernel/git/kdave/btrfs-progs.git ( + http://git.kernel.org/?p=linux/kernel/git/kdave/btrfs-progs.git;a=summary) -- release repository, not for development The **master** branch contains the latest released version and is never rebased. Development git repositories: -* git://github.com/kdave/btrfs-progs.git (`web access `_) -* git://gitlab.com/kdave/btrfs-progs.git (`web access `_) +* git://github.com/kdave/btrfs-progs.git (https://github.com/kdave/btrfs-progs) +* git://gitlab.com/kdave/btrfs-progs.git (https://gitlab.com/kdave/btrfs-progs) For build dependencies and installation instructions please see https://github.com/kdave/btrfs-progs/blob/master/INSTALL diff --git a/Documentation/btrfs-balance.rst b/Documentation/btrfs-balance.rst index 352965bf9d..a33c097367 100644 --- a/Documentation/btrfs-balance.rst +++ b/Documentation/btrfs-balance.rst @@ -248,7 +248,7 @@ This should lead to decrease in the *total* numbers in the **btrfs filesystem df EXIT STATUS ----------- -Unless indicated otherwise below, all **btrfs balance** subcommands +Unless indicated otherwise below, all **btrfs balance** subcommands return a zero exit status if they succeed, and non zero in case of failure. diff --git a/Documentation/ch-balance-examples.rst b/Documentation/ch-balance-examples.rst new file mode 100644 index 0000000000..6a97c33e49 --- /dev/null +++ b/Documentation/ch-balance-examples.rst @@ -0,0 +1,104 @@ + +Adding new device: +***************************** + +The unallocated space requirements depend on the selected storage +profiles. The requirements for the storage profile must be met for the +selected for both data and metadata (e.g. if you have single data and +raid1 metadata, the stricter raid1 requirements must be met or the +filesystem may run out of metadata space and go read-only). + +Before adding a drive, make sure there is enough unallocated space on +existing drives to create new metadata block groups (for filesystems +over 50GB, this is `1GB * (number_of_devices + 2))`. + +If using a striped profile (`raid0`, `raid10`, `raid5`, or `raid6`), then do a +full data balance of all data after adding a drive. If adding multiple +drives at once, do a full data balance after adding the last one. + +.. code-block:: bash + + sudo btrfs balance start -v --full-balance $BTRFS_MOUNT + +If the balance is interrupted, it can be restarted using the 'stripes' +filter (i.e. `-dstripes=1..$N` where $N is the previous size of the array +before the new device was added) as long as all devices are the same size. +If the devices are different sizes, a specialized userspace balance tool +is required. The data balance must be completed before adding any new +devices or increasing the size of existing ones. + +.. code-block:: bash + + ## For going from 4 disk to 5 disks, in Raid 5 + sudo btrfs balance start -v -dstripes=1..4 $BTRFS_MOUNT + +If you are not using a striped profile now, but intend to convert to a +striped profile in the future, always perform a full data balance after +adding drives or replacing existing drives with larger ones. The stock +btrfs balance tool cannot cope with special cases on filesystems with +striped raid profiles, and will paint itself into a corner that will +require custom userspace balancing tools to recover if you try. + +**To watch one can use the following:** + +.. code-block:: bash + + sudo watch "btrfs filesystem usage -T $BTRFS_MOUNT; btrfs balance status $BTRFS_MOUNT" + + +source: + +* https://lore.kernel.org/linux-btrfs/YnmItm%2FNW3eUcvsL@hungrycats.org + +Convert raid1 after mkfs with defaults: +***************************************** +If forgot to set the raid type creating the volume + +.. code-block:: bash + + btrfs balance start -v convert=raid1,soft $BTRFS_MOUNT + +Convert data to raid10 with raid1C4 for metadata: +*************************************** +If you have multi disk setup, or you like to have different profiles on a single disk: + +* ``raid10`` for data +* ``raid1C4`` for metadata and system. + +.. code-block:: bash + + btrfs balance start -v -mconvert=raid1C4,soft -dconvert=raid10,soft $BTRFS_MOUNT + + +Compacts of chunks bellow 50%: +*************************************** +If your data chunks are misbalanced, look at how much space is really used in percentage + +.. code-block:: bash + + btrfs balance start -v -dusage=10 $BTRFS_MOUNT + +and you can feed that to ``-dusage`` in smaller increments starting from 10. +This will ask btrfs to rebalance all chunks that are not at that threshold (bigger number means more work). +Rebalancing means chunks under that usage threshold will have their data moved to other chunks so that they +can be freed up and made available for new allocations (fixing your filesystem full problem). + +.. code-block:: bash + + for USAGE in {10..50..10} do + btrfs balance start -v -dusage=$USAGE $BTRFS_MOUNT + done + +fix incomplete balance: +*********************** + +If the balance is interrupted ( reboot or stopped) during coverting to raid1. +Only targets non `raid1` chunks. + +.. code-block:: bash + + btrfs balance start convert=raid1,soft $BTRFS_MOUNT + +source: + +* https://lore.kernel.org/linux-btrfs/c00a206b-ac20-9312-498f-6fbf1ffd1295@petezilla.co.uk/ diff --git a/Documentation/ch-balance-filters.rst b/Documentation/ch-balance-filters.rst index 2ca141fb5d..1b45cb43c5 100644 --- a/Documentation/ch-balance-filters.rst +++ b/Documentation/ch-balance-filters.rst @@ -1,16 +1,45 @@ -From kernel 3.3 onwards, btrfs balance can limit its action to a subset of the +From kernel 3.3 onwards, BTRFS balance can limit its action to a subset of the whole filesystem, and can be used to change the replication configuration (e.g. -moving data from single to RAID1). This functionality is accessed through the -*-d*, *-m* or *-s* options to btrfs balance start, which filter on data, -metadata and system blocks respectively. +convert data from ``single`` to ``RAID1``). -A filter has the following structure: *type[=params][,type=...]* +Balance can be limited to a group with target block: -The available types are: +* ``-d`` for data blocks. +* ``-m`` for metadata blocks. +* ``-s`` for system blocks. + +To limit filter to a block type, begin filter with the option. Example: ``-dusage=10`` + +Multiple block type can be spesifed per balance. + +A filter has the following structure: ``[target block]filter[=params][,filter=...]`` + +To combind multi filters use ``,``, without spacing. Example: ``-dconvert=raid1,soft`` + +BTRFS can have different profiles on a single disk or the same profile on multiple disks. + +The main reason why you want to have different profiles for data and metadata +is to provide additional protection of the filesystem's metadata when disks fail, +since a single sector of unrecoverable metadata will break the filesystem, +while a single sector of lost data can be trivially recovered by deleting the broken file. + +Before changing profiles, make sure there is enough unallocated space on +existing drives to create new metadata block groups (for filesystems +over 50GB, this is ``1GB * (number_of_devices + 2))``. + +default profile on BTRFS is: + +* Data: ``single`` +* metadata: + * single-disk: ``dup`` + * multiple devices: ``raid1`` + + +The available filter types are: profiles= Balances only block groups with the given profiles. Parameters - are a list of profile names separated by "*|*" (pipe). + are a list of profile names separated by ``|`` (pipe). usage=, usage= Balances only block groups with usage under the given percentage. The @@ -18,55 +47,55 @@ usage=, usage= should not require any new work space allocated. You may want to use *usage=0* in case balance is returning ENOSPC and your filesystem is not too full. - The argument may be a single value or a range. The single value *N* means *at - most N percent used*, equivalent to *..N* range syntax. Kernels prior to 4.4 + The argument may be a single value or a range. The single value ``N`` means *at + most N percent used*, equivalent to ``..N`` range syntax. Kernels prior to 4.4 accept only the single value format. The minimum range boundary is inclusive, maximum is exclusive. devid= Balances only block groups which have at least one chunk on the given - device. To list devices with ids use **btrfs filesystem show**. + device. To list devices with ids use ``BTRFS filesystem show``. drange= Balance only block groups which overlap with the given byte range on any - device. Use in conjunction with *devid* to filter on a specific device. The - parameter is a range specified as *start..end*. + device. Use in conjunction with ``devid`` to filter on a specific device. The + parameter is a range specified as ``start..end``. vrange= Balance only block groups which overlap with the given byte range in the filesystem's internal virtual address space. This is the address space that most reports from btrfs in the kernel log use. The parameter is a range - specified as *start..end*. + specified as ``start..end``. convert= Convert each selected block group to the given profile name identified by parameters. .. note:: - Starting with kernel 4.5, the *data* chunks can be converted to/from the - *DUP* profile on a single device. + Starting with kernel 4.5, the ``data`` chunks can be converted to/from the + ``DUP`` profile on a single device. .. note:: - Starting with kernel 4.6, all profiles can be converted to/from *DUP* on + Starting with kernel 4.6, all profiles can be converted to/from ``DUP`` on multi-device filesystems. limit=, limit= Process only given number of chunks, after all filters are applied. This can be - used to specifically target a chunk in connection with other filters (*drange*, - *vrange*) or just simply limit the amount of work done by a single balance run. + used to specifically target a chunk in connection with other filters (``drange``, + ``vrange``) or just simply limit the amount of work done by a single balance run. - The argument may be a single value or a range. The single value *N* means *at - most N chunks*, equivalent to *..N* range syntax. Kernels prior to 4.4 accept + The argument may be a single value or a range. The single value ``N`` means *at + most N chunks*, equivalent to ``..N`` range syntax. Kernels prior to 4.4 accept only the single value format. The range minimum and maximum are inclusive. stripes= Balance only block groups which have the given number of stripes. The parameter - is a range specified as *start..end*. Makes sense for block group profiles that + is a range specified as ``start..end``. Makes sense for block group profiles that utilize striping, ie. RAID0/10/5/6. The range minimum and maximum are inclusive. soft - Takes no parameters. Only has meaning when converting between profiles. + Takes no parameters. Only has meaning when converting between profiles, or When doing convert from one profile to another and soft mode is on, chunks that already have the target profile are left untouched. This is useful e.g. when half of the filesystem was converted earlier but got @@ -76,8 +105,18 @@ soft For example, this means that we can convert metadata chunks the "hard" way while converting data chunks selectively with soft switch. -Profile names, used in *profiles* and *convert* are one of: *raid0*, *raid1*, -*raid1c3*, *raid1c4*, *raid10*, *raid5*, *raid6*, *dup*, *single*. The mixed -data/metadata profiles can be converted in the same way, but it's conversion +Profile names, used in ``profiles`` and ``convert`` are one of: + +* ``raid0`` +* ``raid1`` +* ``raid1c3`` +* ``raid1c4`` +* ``raid10`` +* ``raid5`` +* ``raid6`` +* ``dup`` +* ``single`` + +The mixed data/metadata profiles can be converted in the same way, but it's conversion between mixed and non-mixed is not implemented. For the constraints of the profiles please refer to ``mkfs.btrfs(8)``, section *PROFILES*. diff --git a/Documentation/index.rst b/Documentation/index.rst index ca812b1b5e..2ae609dabe 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -12,6 +12,7 @@ Welcome to BTRFS documentation! Administration Hardware CHANGES + Feature-by-version Glossary INSTALL @@ -43,13 +44,20 @@ Welcome to BTRFS documentation! Volume-management Zoned-mode +.. toctree:: + :maxdepth: 1 + :caption: Project information + + Source-repositories + Contributors + .. toctree:: :maxdepth: 1 :caption: TODO Quick-start Interoperability - project-index trouble-index Experimental btrfs-ioctl + DocConventions diff --git a/Documentation/project-index.rst b/Documentation/project-index.rst deleted file mode 100644 index 6a4c285126..0000000000 --- a/Documentation/project-index.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. BTRFS project related pages index - -Project pages -============= - -.. toctree:: - :maxdepth: 1 - - Source-repositories diff --git a/Documentation/trouble-index.rst b/Documentation/trouble-index.rst index 97eb1f0f8a..42d31d3518 100644 --- a/Documentation/trouble-index.rst +++ b/Documentation/trouble-index.rst @@ -3,7 +3,8 @@ Troubleshooting pages ===================== -Correctness related, permanent +System messages printed to the log (dmesg, syslog, journal) have limited space +for description and may need further explanation what needs to be done. Error: parent transid verify error ---------------------------------- @@ -94,6 +95,97 @@ device that's available for the filesystem but without any other existing block groups. Before balance starts a check is performed to verify the requested action is possible. If not, ENOSPC is returned. +Error: unable to start balance with target metadata profile +----------------------------------------------------------- + +.. code-block:: + + unable to start balance with target metadata profile 32 + +This means that a conversion has been attempted from profile *RAID1* to *dup* +with btrfs-progs earlier than version 4.7. Update and you'll be able to do the +conversion. + +Error: balance will reduce metadata integrity +--------------------------------------------- + +The full message in system log + +.. code-block:: + + balance will reduce metadata integrity, use force if you want this + +This means that conversion will remove a degree of metadata redundancy, for +example when going from profile *RAID1* or *dup* to *single*. The force +parameter to ``btrfs balance start -f`` is needed. + +How to clean old super block +---------------------------- + +The preferred way is to use the ``wipefs`` utility that is part of the +*util-linux* package. Running the command with the device will not destroy +the data, just list the detected filesystems: + +.. code-block:: + + # wipefs /dev/sda + offset type + ---------------------------------------------------------------- + 0x10040 btrfs [filesystem] + UUID: 7760469b-1704-487e-9b96-7d7a57d218a5 + +Remove the filesystem signature at a given offset or wipe all recognized +signatures on the device: + +.. code-block:: + + # wipefs -o 0x10040 /dev/sda + 8 bytes [5f 42 48 52 66 53 5f 4d] erased at offset 0x10040 (btrfs) + + # wipefs -a /dev/sda + 8 bytes [5f 42 48 52 66 53 5f 4d] erased at offset 0x10040 (btrfs) + +.. note:: + + The process is reversible, if the 8 bytes are written back, the device is + recognized again. See below. + +.. note:: + + *wipefs* clears only the first super block. If available, the second and + third copies can be used to resurrect the filesystem. + +Stale signature on device +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Related problem regarding partitioned and unpartitioned device: *Long time ago +I created btrfs on /dev/sda. After some changes btrfs moved to /dev/sda1.* + +Use ``wipefs -o 0x10040`` (ie. with the offset of the btrfs signature), it +won't touch the parition table. + +Manual deletion of super block signature +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are three superblocks: the first one is located at 64KiB, the second one +at 64MiB, the third one at 256GiB. The following lines reset the signature +on all the three copies: + + +.. code-block:: + + # dd if=/dev/zero bs=1 count=8 of=/dev/sda seek=$((64*1024+64)) + # dd if=/dev/zero bs=1 count=8 of=/dev/sda seek=$((64*1024*1024+64)) + # dd if=/dev/zero bs=1 count=8 of=/dev/sda seek=$((256*1024*1024*1024+64)) + +If you want to restore the super block signatures: + +.. code-block:: + + # echo "_BHRfS_M" | dd bs=1 count=8 of=/dev/sda seek=$((64*1024+64)) + # echo "_BHRfS_M" | dd bs=1 count=8 of=/dev/sda seek=$((64*1024*1024+64)) + # echo "_BHRfS_M" | dd bs=1 count=8 of=/dev/sda seek=$((256*1024*1024*1024+64)) + Generic errors, errno --------------------- diff --git a/Makefile b/Makefile index ea5a1ae86f..2a37d1c6b5 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,8 @@ CFLAGS = $(SUBST_CFLAGS) \ -I$(TOPDIR) \ -I$(TOPDIR)/libbtrfsutil \ $(CRYPTO_CFLAGS) \ + -DCOMPRESSION_LZO=$(COMPRESSION_LZO) \ + -DCOMPRESSION_ZSTD=$(COMPRESSION_ZSTD) \ $(DISABLE_WARNING_FLAGS) \ $(ENABLE_WARNING_FLAGS) \ $(EXTRAWARN_CFLAGS) \ @@ -334,7 +336,7 @@ endif btrfs_convert_cflags = -DBTRFSCONVERT_EXT2=$(BTRFSCONVERT_EXT2) btrfs_convert_cflags += -DBTRFSCONVERT_REISERFS=$(BTRFSCONVERT_REISERFS) btrfs_fragments_libs = -lgd -lpng -ljpeg -lfreetype -cmds_restore_cflags = -DBTRFSRESTORE_LZO=$(BTRFSRESTORE_LZO) -DBTRFSRESTORE_ZSTD=$(BTRFSRESTORE_ZSTD) +cmds_restore_cflags = -DCOMPRESSION_LZO=$(COMPRESSION_LZO) -DCOMPRESSION_ZSTD=$(COMPRESSION_ZSTD) ifeq ($(CRYPTOPROVIDER_BUILTIN),1) CRYPTO_OBJECTS = crypto/sha224-256.o crypto/blake2b-ref.o diff --git a/Makefile.inc.in b/Makefile.inc.in index 385b7ae1cf..d832c2db10 100644 --- a/Makefile.inc.in +++ b/Makefile.inc.in @@ -16,9 +16,9 @@ BUILD_PROGRAMS = @BUILD_PROGRAMS@ BUILD_SHARED_LIBRARIES = @BUILD_SHARED_LIBRARIES@ BUILD_STATIC_LIBRARIES = @BUILD_STATIC_LIBRARIES@ BTRFSCONVERT_EXT2 = @BTRFSCONVERT_EXT2@ -BTRFSRESTORE_LZO = @BTRFSRESTORE_LZO@ BTRFSCONVERT_REISERFS = @BTRFSCONVERT_REISERFS@ -BTRFSRESTORE_ZSTD = @BTRFSRESTORE_ZSTD@ +COMPRESSION_LZO = @COMPRESSION_LZO@ +COMPRESSION_ZSTD = @COMPRESSION_ZSTD@ PYTHON_BINDINGS = @PYTHON_BINDINGS@ PYTHON = @PYTHON@ PYTHON_CFLAGS = @PYTHON_CFLAGS@ diff --git a/btrfs-corrupt-block.c b/btrfs-corrupt-block.c index e961255d5d..22b2a5eeef 100644 --- a/btrfs-corrupt-block.c +++ b/btrfs-corrupt-block.c @@ -1236,7 +1236,7 @@ int main(int argc, char **argv) while(1) { int c; - enum { GETOPT_VAL_BLOCK_GROUP = 256 }; + enum { GETOPT_VAL_BLOCK_GROUP = GETOPT_VAL_FIRST }; static const struct option long_options[] = { /* { "byte-count", 1, NULL, 'b' }, */ { "logical", required_argument, NULL, 'l' }, diff --git a/btrfstune.c b/btrfstune.c index 31be907486..d1a1877ee4 100644 --- a/btrfstune.c +++ b/btrfstune.c @@ -818,7 +818,7 @@ int BOX_MAIN(btrfstune)(int argc, char *argv[]) int fd = -1; while(1) { - enum { GETOPT_VAL_CSUM = 256 }; + enum { GETOPT_VAL_CSUM = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "help", no_argument, NULL, GETOPT_VAL_HELP}, #if EXPERIMENTAL diff --git a/check/main.c b/check/main.c index 6b2be57a15..4f7ab8b293 100644 --- a/check/main.c +++ b/check/main.c @@ -10412,7 +10412,7 @@ static int cmd_check(const struct cmd_struct *cmd, int argc, char **argv) while(1) { int c; - enum { GETOPT_VAL_REPAIR = 257, GETOPT_VAL_INIT_CSUM, + enum { GETOPT_VAL_REPAIR = GETOPT_VAL_FIRST, GETOPT_VAL_INIT_CSUM, GETOPT_VAL_INIT_EXTENT, GETOPT_VAL_CHECK_CSUM, GETOPT_VAL_READONLY, GETOPT_VAL_CHUNK_TREE, GETOPT_VAL_MODE, GETOPT_VAL_CLEAR_SPACE_CACHE, diff --git a/cmds/balance.c b/cmds/balance.c index 7abc69d926..de4085799d 100644 --- a/cmds/balance.c +++ b/cmds/balance.c @@ -407,9 +407,8 @@ static int cmd_balance_start(const struct cmd_struct *cmd, optind = 0; while (1) { - enum { GETOPT_VAL_FULL_BALANCE = 256, - GETOPT_VAL_BACKGROUND = 257, - GETOPT_VAL_ENQUEUE }; + enum { GETOPT_VAL_FULL_BALANCE = GETOPT_VAL_FIRST, + GETOPT_VAL_BACKGROUND, GETOPT_VAL_ENQUEUE }; static const struct option longopts[] = { { "data", optional_argument, NULL, 'd'}, { "metadata", optional_argument, NULL, 'm' }, diff --git a/cmds/device.c b/cmds/device.c index 7d3febff96..a945adafe3 100644 --- a/cmds/device.c +++ b/cmds/device.c @@ -74,7 +74,7 @@ static int cmd_device_add(const struct cmd_struct *cmd, optind = 0; while (1) { int c; - enum { GETOPT_VAL_ENQUEUE = 256 }; + enum { GETOPT_VAL_ENQUEUE = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "nodiscard", optional_argument, NULL, 'K'}, { "force", no_argument, NULL, 'f'}, @@ -199,7 +199,7 @@ static int _cmd_device_remove(const struct cmd_struct *cmd, optind = 0; while (1) { int c; - enum { GETOPT_VAL_ENQUEUE = 256 }; + enum { GETOPT_VAL_ENQUEUE = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "enqueue", no_argument, NULL, GETOPT_VAL_ENQUEUE}, { NULL, 0, NULL, 0} diff --git a/cmds/inspect-dump-super.c b/cmds/inspect-dump-super.c index d8435624df..58f9ba5f0d 100644 --- a/cmds/inspect-dump-super.c +++ b/cmds/inspect-dump-super.c @@ -92,7 +92,7 @@ static int cmd_inspect_dump_super(const struct cmd_struct *cmd, while (1) { int c; - enum { GETOPT_VAL_BYTENR = 257 }; + enum { GETOPT_VAL_BYTENR = GETOPT_VAL_FIRST }; static const struct option long_options[] = { {"all", no_argument, NULL, 'a'}, {"bytenr", required_argument, NULL, GETOPT_VAL_BYTENR }, diff --git a/cmds/inspect-dump-tree.c b/cmds/inspect-dump-tree.c index daa7f92551..73ffd57eb1 100644 --- a/cmds/inspect-dump-tree.c +++ b/cmds/inspect-dump-tree.c @@ -342,7 +342,8 @@ static int cmd_inspect_dump_tree(const struct cmd_struct *cmd, optind = 0; while (1) { int c; - enum { GETOPT_VAL_FOLLOW = 256, GETOPT_VAL_DFS, GETOPT_VAL_BFS, + enum { GETOPT_VAL_FOLLOW = GETOPT_VAL_FIRST, GETOPT_VAL_DFS, + GETOPT_VAL_BFS, GETOPT_VAL_NOSCAN, GETOPT_VAL_HIDE_NAMES, GETOPT_VAL_CSUM_HEADERS, GETOPT_VAL_CSUM_ITEMS, }; diff --git a/cmds/inspect.c b/cmds/inspect.c index 1534f2040f..19909f8713 100644 --- a/cmds/inspect.c +++ b/cmds/inspect.c @@ -648,7 +648,7 @@ static int cmd_inspect_min_dev_size(const struct cmd_struct *cmd, optind = 0; while (1) { int c; - enum { GETOPT_VAL_DEVID = 256 }; + enum { GETOPT_VAL_DEVID = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "id", required_argument, NULL, GETOPT_VAL_DEVID }, {NULL, 0, NULL, 0} diff --git a/cmds/qgroup.c b/cmds/qgroup.c index 139dca970f..8e044a60a6 100644 --- a/cmds/qgroup.c +++ b/cmds/qgroup.c @@ -1535,7 +1535,7 @@ static int _cmd_qgroup_assign(const struct cmd_struct *cmd, int assign, optind = 0; while (1) { - enum { GETOPT_VAL_RESCAN = 256, GETOPT_VAL_NO_RESCAN }; + enum { GETOPT_VAL_RESCAN = GETOPT_VAL_FIRST, GETOPT_VAL_NO_RESCAN }; static const struct option long_options[] = { { "rescan", no_argument, NULL, GETOPT_VAL_RESCAN }, { "no-rescan", no_argument, NULL, GETOPT_VAL_NO_RESCAN }, @@ -1757,7 +1757,7 @@ static int cmd_qgroup_show(const struct cmd_struct *cmd, int argc, char **argv) while (1) { int c; enum { - GETOPT_VAL_SORT = 256, + GETOPT_VAL_SORT = GETOPT_VAL_FIRST, GETOPT_VAL_SYNC }; static const struct option long_options[] = { diff --git a/cmds/receive-dump.c b/cmds/receive-dump.c index df5991e1da..92e0a4c9a7 100644 --- a/cmds/receive-dump.c +++ b/cmds/receive-dump.c @@ -339,9 +339,9 @@ static int print_fallocate(const char *path, int mode, u64 offset, u64 len, mode, offset, len); } -static int print_setflags(const char *path, int flags, void *user) +static int print_fileattr(const char *path, u64 attr, void *user) { - return PRINT_DUMP(user, path, "setflags", "flags=%d", flags); + return PRINT_DUMP(user, path, "fileattr", "fileattr=0x%llu", attr); } struct btrfs_send_ops btrfs_print_send_ops = { @@ -368,5 +368,5 @@ struct btrfs_send_ops btrfs_print_send_ops = { .update_extent = print_update_extent, .encoded_write = print_encoded_write, .fallocate = print_fallocate, - .setflags = print_setflags, + .fileattr = print_fileattr, }; diff --git a/cmds/receive.c b/cmds/receive.c index 7f76a04f05..aec3245871 100644 --- a/cmds/receive.c +++ b/cmds/receive.c @@ -41,9 +41,14 @@ #include #include -#include #include +#if COMPRESSION_LZO +#include +#include +#endif +#if COMPRESSION_ZSTD #include +#endif #include "kernel-shared/ctree.h" #include "ioctl.h" @@ -70,7 +75,6 @@ struct btrfs_receive char *dest_dir_path; /* relative to root_path */ char full_subvol_path[PATH_MAX]; char *full_root_path; - int dest_dir_chroot; struct subvol_info cur_subvol; /* @@ -79,12 +83,16 @@ struct btrfs_receive */ char cur_subvol_path[PATH_MAX]; - int honor_end_cmd; + bool dest_dir_chroot; + + bool honor_end_cmd; bool force_decompress; +#if COMPRESSION_ZSTD /* Reuse stream objects for encoded_write decompression fallback */ ZSTD_DStream *zstd_dstream; +#endif z_stream *zlib_stream; }; @@ -1042,6 +1050,7 @@ static int decompress_zlib(struct btrfs_receive *rctx, const char *encoded_data, return 0; } +#if COMPRESSION_ZSTD static int decompress_zstd(struct btrfs_receive *rctx, const char *encoded_buf, u64 encoded_len, char *unencoded_buf, u64 unencoded_len) @@ -1080,7 +1089,9 @@ static int decompress_zstd(struct btrfs_receive *rctx, const char *encoded_buf, } return 0; } +#endif +#if COMPRESSION_LZO static int decompress_lzo(const char *encoded_data, u64 encoded_len, char *unencoded_data, u64 unencoded_len, unsigned int sector_size) @@ -1142,6 +1153,7 @@ static int decompress_lzo(const char *encoded_data, u64 encoded_len, } return 0; } +#endif static int decompress_and_write(struct btrfs_receive *rctx, const char *encoded_data, u64 offset, @@ -1153,7 +1165,7 @@ static int decompress_and_write(struct btrfs_receive *rctx, size_t pos; ssize_t w; char *unencoded_data; - int sector_shift; + int sector_shift = 0; unencoded_data = calloc(unencoded_len, 1); if (!unencoded_data) { @@ -1168,17 +1180,24 @@ static int decompress_and_write(struct btrfs_receive *rctx, if (ret) goto out; break; +#if COMPRESSION_ZSTD case BTRFS_ENCODED_IO_COMPRESSION_ZSTD: ret = decompress_zstd(rctx, encoded_data, encoded_len, unencoded_data, unencoded_len); if (ret) goto out; break; +#else + error("ZSTD compression for stream not compiled in"); + ret = -EOPNOTSUPP; + goto out; +#endif case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K: case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K: case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K: case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K: case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K: +#if COMPRESSION_LZO sector_shift = compression - BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12; ret = decompress_lzo(encoded_data, encoded_len, unencoded_data, @@ -1186,6 +1205,11 @@ static int decompress_and_write(struct btrfs_receive *rctx, if (ret) goto out; break; +#else + error("LZO compression for stream not compiled in"); + ret = -EOPNOTSUPP; + goto out; +#endif default: error("unknown compression: %d", compression); ret = -EOPNOTSUPP; @@ -1285,7 +1309,7 @@ static int process_fallocate(const char *path, int mode, u64 offset, u64 len, return 0; } -static int process_setflags(const char *path, int flags, void *user) +static int process_fileattr(const char *path, u64 attr, void *user) { int ret; struct btrfs_receive *rctx = user; @@ -1293,16 +1317,17 @@ static int process_setflags(const char *path, int flags, void *user) ret = path_cat_out(full_path, rctx->full_subvol_path, path); if (ret < 0) { - error("setflags: path invalid: %s", path); + error("fileattr: path invalid: %s", path); return ret; } ret = open_inode_for_write(rctx, full_path); if (ret < 0) return ret; - ret = ioctl(rctx->write_fd, FS_IOC_SETFLAGS, &flags); + ret = -EOPNOTSUPP; + /* ret = ioctl(rctx->write_fd, FS_IOC_SETFLAGS, &flags); */ if (ret < 0) { ret = -errno; - error("setflags: setflags ioctl on %s failed: %m", path); + error("fileattr: set file attributes on %s failed: %m", path); return ret; } return 0; @@ -1332,7 +1357,7 @@ static struct btrfs_send_ops send_ops = { .update_extent = process_update_extent, .encoded_write = process_encoded_write, .fallocate = process_fallocate, - .setflags = process_setflags, + .fileattr = process_fileattr, }; static int do_receive(struct btrfs_receive *rctx, const char *tomnt, @@ -1485,8 +1510,10 @@ static int do_receive(struct btrfs_receive *rctx, const char *tomnt, close(rctx->dest_dir_fd); rctx->dest_dir_fd = -1; } +#if COMPRESSION_ZSTD if (rctx->zstd_dstream) ZSTD_freeDStream(rctx->zstd_dstream); +#endif if (rctx->zlib_stream) { inflateEnd(rctx->zlib_stream); free(rctx->zlib_stream); @@ -1530,6 +1557,15 @@ static const char * const cmd_receive_usage[] = { HELPINFO_INSERT_GLOBALS, HELPINFO_INSERT_VERBOSE, HELPINFO_INSERT_QUIET, + "", + "Compression support: zlib" +#if COMPRESSION_LZO + ", lzo" +#endif +#if COMPRESSION_ZSTD + ", zstd" +#endif + , NULL }; @@ -1548,7 +1584,7 @@ static int cmd_receive(const struct cmd_struct *cmd, int argc, char **argv) rctx.mnt_fd = -1; rctx.write_fd = -1; rctx.dest_dir_fd = -1; - rctx.dest_dir_chroot = 0; + rctx.dest_dir_chroot = false; realmnt[0] = 0; fromfile[0] = 0; @@ -1568,7 +1604,7 @@ static int cmd_receive(const struct cmd_struct *cmd, int argc, char **argv) while (1) { int c; enum { - GETOPT_VAL_DUMP = 257, + GETOPT_VAL_DUMP = GETOPT_VAL_FIRST, GETOPT_VAL_FORCE_DECOMPRESS, }; static const struct option long_opts[] = { @@ -1600,10 +1636,10 @@ static int cmd_receive(const struct cmd_struct *cmd, int argc, char **argv) } break; case 'e': - rctx.honor_end_cmd = 1; + rctx.honor_end_cmd = true; break; case 'C': - rctx.dest_dir_chroot = 1; + rctx.dest_dir_chroot = true; break; case 'E': max_errors = arg_strtou64(optarg); diff --git a/cmds/replace.c b/cmds/replace.c index 23f5dccaf4..dd15dabae8 100644 --- a/cmds/replace.c +++ b/cmds/replace.c @@ -147,7 +147,7 @@ static int cmd_replace_start(const struct cmd_struct *cmd, optind = 0; while (1) { int c; - enum { GETOPT_VAL_ENQUEUE = 256 }; + enum { GETOPT_VAL_ENQUEUE = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "enqueue", no_argument, NULL, GETOPT_VAL_ENQUEUE}, { "nodiscard", no_argument, NULL, 'K' }, diff --git a/cmds/restore.c b/cmds/restore.c index 885877c531..b517c30c9f 100644 --- a/cmds/restore.c +++ b/cmds/restore.c @@ -25,12 +25,12 @@ #include #include #include -#if BTRFSRESTORE_LZO +#if COMPRESSION_LZO #include #include #endif #include -#if BTRFSRESTORE_ZSTD +#if COMPRESSION_ZSTD #include #endif #include @@ -100,7 +100,7 @@ static inline size_t read_compress_length(unsigned char *buf) static int decompress_lzo(struct btrfs_root *root, unsigned char *inbuf, char *outbuf, u64 compress_len, u64 *decompress_len) { -#if !BTRFSRESTORE_LZO +#if !COMPRESSION_LZO error("btrfs-restore not compiled with lzo support"); return -1; #else @@ -168,7 +168,7 @@ static int decompress_lzo(struct btrfs_root *root, unsigned char *inbuf, static int decompress_zstd(const char *inbuf, char *outbuf, u64 compress_len, u64 decompress_len) { -#if !BTRFSRESTORE_ZSTD +#if !COMPRESSION_ZSTD error("btrfs not compiled with zstd support"); return -1; #else @@ -1360,10 +1360,10 @@ static const char * const cmd_restore_usage[] = { HELPINFO_INSERT_VERBOSE, "", "Compression support: zlib" -#if BTRFSRESTORE_LZO +#if COMPRESSION_LZO ", lzo" #endif -#if BTRFSRESTORE_ZSTD +#if COMPRESSION_ZSTD ", zstd" #endif , @@ -1391,7 +1391,7 @@ static int cmd_restore(const struct cmd_struct *cmd, int argc, char **argv) optind = 0; while (1) { int opt; - enum { GETOPT_VAL_PATH_REGEX = 256 }; + enum { GETOPT_VAL_PATH_REGEX = GETOPT_VAL_FIRST }; static const struct option long_options[] = { { "path-regex", required_argument, NULL, GETOPT_VAL_PATH_REGEX }, diff --git a/cmds/send.c b/cmds/send.c index b1adfeca5b..f944630256 100644 --- a/cmds/send.c +++ b/cmds/send.c @@ -538,7 +538,7 @@ static int cmd_send(const struct cmd_struct *cmd, int argc, char **argv) optind = 0; while (1) { enum { - GETOPT_VAL_SEND_NO_DATA = 256, + GETOPT_VAL_SEND_NO_DATA = GETOPT_VAL_FIRST, GETOPT_VAL_PROTO, GETOPT_VAL_COMPRESSED_DATA, }; diff --git a/common/help.h b/common/help.h index ea0552500f..448e51c9aa 100644 --- a/common/help.h +++ b/common/help.h @@ -17,16 +17,19 @@ #ifndef __BTRFS_HELP_H__ #define __BTRFS_HELP_H__ -#define GETOPT_VAL_SI 256 -#define GETOPT_VAL_IEC 257 -#define GETOPT_VAL_RAW 258 -#define GETOPT_VAL_HUMAN_READABLE 259 -#define GETOPT_VAL_KBYTES 260 -#define GETOPT_VAL_MBYTES 261 -#define GETOPT_VAL_GBYTES 262 -#define GETOPT_VAL_TBYTES 263 - -#define GETOPT_VAL_HELP 270 +/* User defined long options first option */ +#define GETOPT_VAL_FIRST 256 + +#define GETOPT_VAL_SI 512 +#define GETOPT_VAL_IEC 513 +#define GETOPT_VAL_RAW 514 +#define GETOPT_VAL_HUMAN_READABLE 515 +#define GETOPT_VAL_KBYTES 516 +#define GETOPT_VAL_MBYTES 517 +#define GETOPT_VAL_GBYTES 518 +#define GETOPT_VAL_TBYTES 519 + +#define GETOPT_VAL_HELP 520 #define ARGV0_BUF_SIZE PATH_MAX diff --git a/common/send-stream.c b/common/send-stream.c index 0fc93463e0..96d1aa218d 100644 --- a/common/send-stream.c +++ b/common/send-stream.c @@ -372,10 +372,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) u64 unencoded_file_len; u64 unencoded_len; u64 unencoded_offset; + u64 fileattr; int len; int xattr_len; int fallocate_mode; - int setflags_flags; ret = read_cmd(sctx); if (ret) @@ -548,10 +548,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx) ret = sctx->ops->fallocate(path, fallocate_mode, offset, tmp, sctx->user); break; - case BTRFS_SEND_C_SETFLAGS: + case BTRFS_SEND_C_FILEATTR: TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path); - TLV_GET_U32(sctx, BTRFS_SEND_A_SETFLAGS_FLAGS, &setflags_flags); - ret = sctx->ops->setflags(path, setflags_flags, sctx->user); + TLV_GET_U32(sctx, BTRFS_SEND_A_FILEATTR, &fileattr); + ret = sctx->ops->fileattr(path, fileattr, sctx->user); break; } diff --git a/common/send-stream.h b/common/send-stream.h index 3189f8897e..b5973b66fa 100644 --- a/common/send-stream.h +++ b/common/send-stream.h @@ -59,7 +59,7 @@ struct btrfs_send_ops { u32 encryption, void *user); int (*fallocate)(const char *path, int mode, u64 offset, u64 len, void *user); - int (*setflags)(const char *path, int flags, void *user); + int (*fileattr)(const char *path, u64 attr, void *user); }; int btrfs_read_and_process_send_stream(int fd, diff --git a/configure.ac b/configure.ac index fa19015b5f..b0b5547cb1 100644 --- a/configure.ac +++ b/configure.ac @@ -313,7 +313,7 @@ PKG_CHECK_MODULES(ZLIB, [zlib]) PKG_STATIC(ZLIB_LIBS_STATIC, [zlib]) AC_ARG_ENABLE([zstd], - AS_HELP_STRING([--disable-zstd], [build without zstd support]), + AS_HELP_STRING([--disable-zstd], [build without zstd support for restore and receive (default: enabled)]), [], [enable_zstd=yes] ) @@ -322,8 +322,8 @@ if test "x$enable_zstd" = xyes; then PKG_STATIC(ZSTD_LIBS_STATIC, [libzstd]) fi -AS_IF([test "x$enable_zstd" = xyes], [BTRFSRESTORE_ZSTD=1], [BTRFSRESTORE_ZSTD=0]) -AC_SUBST(BTRFSRESTORE_ZSTD) +AS_IF([test "x$enable_zstd" = xyes], [COMPRESSION_ZSTD=1], [COMPRESSION_ZSTD=0]) +AC_SUBST(COMPRESSION_ZSTD) AC_ARG_ENABLE([libudev], AS_HELP_STRING([--disable-libudev], [build without libudev support (for multipath)]), @@ -373,7 +373,7 @@ fi AC_SUBST(UDEVDIR) AC_ARG_ENABLE([lzo], - AS_HELP_STRING([--disable-lzo], [build without lzo support (default: enabled)]), + AS_HELP_STRING([--disable-lzo], [build without lzo support for restore and receive (default: enabled)]), [], [enable_lzo=yes] ) @@ -390,8 +390,8 @@ if test "x$enable_lzo" = xyes; then AC_SUBST([LZO2_CFLAGS]) fi -AS_IF([test "x$enable_lzo" = xyes], [BTRFSRESTORE_LZO=1], [BTRFSRESTORE_LZO=0]) -AC_SUBST(BTRFSRESTORE_LZO) +AS_IF([test "x$enable_lzo" = xyes], [COMPRESSION_LZO=1], [COMPRESSION_LZO=0]) +AC_SUBST(COMPRESSION_LZO) dnl call PKG_INSTALLDIR from pkg.m4 to set pkgconfigdir m4_ifdef([PKG_INSTALLDIR], [PKG_INSTALLDIR], [AC_MSG_ERROR([please install pkgconf])]) @@ -445,7 +445,8 @@ AC_MSG_RESULT([ doc generator: ${DOC_TOOL} backtrace support: ${enable_backtrace} btrfs-convert: ${enable_convert} ${convertfs:+($convertfs)} - btrfs-restore zstd: ${enable_zstd} + zstd support: ${enable_zstd} + lzo support: ${enable_lzo} Python bindings: ${enable_python} Python interpreter: ${PYTHON} crypto provider: ${cryptoprovider} ${cryptoproviderversion} diff --git a/convert/main.c b/convert/main.c index 73f919d252..95ceec1123 100644 --- a/convert/main.c +++ b/convert/main.c @@ -1832,7 +1832,7 @@ int BOX_MAIN(convert)(int argc, char *argv[]) printf("btrfs-convert from %s\n\n", PACKAGE_STRING); while(1) { - enum { GETOPT_VAL_NO_PROGRESS = 256, GETOPT_VAL_CHECKSUM, + enum { GETOPT_VAL_NO_PROGRESS = GETOPT_VAL_FIRST, GETOPT_VAL_CHECKSUM, GETOPT_VAL_UUID }; static const struct option long_options[] = { { "no-progress", no_argument, NULL, diff --git a/kernel-shared/send.h b/kernel-shared/send.h index 1378c72b32..0236d9fd8b 100644 --- a/kernel-shared/send.h +++ b/kernel-shared/send.h @@ -98,7 +98,7 @@ enum btrfs_send_cmd { /* Version 2 */ BTRFS_SEND_C_FALLOCATE = 23, - BTRFS_SEND_C_SETFLAGS = 24, + BTRFS_SEND_C_FILEATTR = 24, BTRFS_SEND_C_ENCODED_WRITE = 25, BTRFS_SEND_C_MAX_V2 = 25, @@ -151,7 +151,13 @@ enum { /* Version 2 */ BTRFS_SEND_A_FALLOCATE_MODE = 25, - BTRFS_SEND_A_SETFLAGS_FLAGS = 26, + /* + * File attributes from the FS_*_FL namespace (i_flags, xflags), + * translated to BTRFS_INODE_* bits (BTRFS_INODE_FLAG_MASK) and stored + * in btrfs_inode_item::flags (represented by btrfs_inode::flags and + * btrfs_inode::ro_flags). + */ + BTRFS_SEND_A_FILEATTR = 26, BTRFS_SEND_A_UNENCODED_FILE_LEN = 27, BTRFS_SEND_A_UNENCODED_LEN = 28, diff --git a/libbtrfs/ctree.h b/libbtrfs/ctree.h index 5374dc5de9..9279c4161d 100644 --- a/libbtrfs/ctree.h +++ b/libbtrfs/ctree.h @@ -2653,149 +2653,6 @@ static inline int __btrfs_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag) ((unsigned long)(btrfs_leaf_data(leaf) + \ btrfs_item_offset(leaf, slot))) -u64 btrfs_name_hash(const char *name, int len); -u64 btrfs_extref_hash(u64 parent_objectid, const char *name, int len); - -/* extent-tree.c */ -int btrfs_reserve_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 num_bytes, u64 empty_size, - u64 hint_byte, u64 search_end, - struct btrfs_key *ins, bool is_data); -int btrfs_fix_block_accounting(struct btrfs_trans_handle *trans); -void btrfs_pin_extent(struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes); -void btrfs_unpin_extent(struct btrfs_fs_info *fs_info, - u64 bytenr, u64 num_bytes); -struct btrfs_block_group *btrfs_lookup_block_group(struct btrfs_fs_info *info, - u64 bytenr); -struct btrfs_block_group *btrfs_lookup_first_block_group(struct - btrfs_fs_info *info, - u64 bytenr); -struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u32 blocksize, u64 root_objectid, - struct btrfs_disk_key *key, int level, - u64 hint, u64 empty_size); -int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, - struct btrfs_fs_info *fs_info, u64 bytenr, - u64 offset, int metadata, u64 *refs, u64 *flags); -int btrfs_set_block_flags(struct btrfs_trans_handle *trans, u64 bytenr, - int level, u64 flags); -int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, - struct extent_buffer *buf, int record_parent); -int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, - struct extent_buffer *buf, int record_parent); -int btrfs_free_tree_block(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct extent_buffer *buf, - u64 parent, int last_ref); -int btrfs_free_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 bytenr, u64 num_bytes, u64 parent, - u64 root_objectid, u64 owner, u64 offset); -void btrfs_finish_extent_commit(struct btrfs_trans_handle *trans); -int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 bytenr, u64 num_bytes, u64 parent, - u64 root_objectid, u64 owner, u64 offset); -int btrfs_update_extent_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, u64 bytenr, - u64 orig_parent, u64 parent, - u64 root_objectid, u64 ref_generation, - u64 owner_objectid); -int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans); -int update_space_info(struct btrfs_fs_info *info, u64 flags, - u64 total_bytes, u64 bytes_used, - struct btrfs_space_info **space_info); -int btrfs_free_block_groups(struct btrfs_fs_info *info); -int btrfs_read_block_groups(struct btrfs_fs_info *info); -struct btrfs_block_group * -btrfs_add_block_group(struct btrfs_fs_info *fs_info, u64 bytes_used, u64 type, - u64 chunk_offset, u64 size); -int btrfs_make_block_group(struct btrfs_trans_handle *trans, - struct btrfs_fs_info *fs_info, u64 bytes_used, - u64 type, u64 chunk_offset, u64 size); -int btrfs_make_block_groups(struct btrfs_trans_handle *trans, - struct btrfs_fs_info *fs_info); -int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, - u64 num, int alloc, int mark_free); -int btrfs_record_file_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, u64 objectid, - struct btrfs_inode_item *inode, - u64 file_pos, u64 disk_bytenr, - u64 num_bytes); -int btrfs_remove_block_group(struct btrfs_trans_handle *trans, - u64 bytenr, u64 len); -void free_excluded_extents(struct btrfs_fs_info *fs_info, - struct btrfs_block_group *cache); -int exclude_super_stripes(struct btrfs_fs_info *fs_info, - struct btrfs_block_group *cache); -u64 add_new_free_space(struct btrfs_block_group *block_group, - struct btrfs_fs_info *info, u64 start, u64 end); -u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset); - -/* ctree.c */ -int btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2); -int btrfs_del_ptr(struct btrfs_root *root, struct btrfs_path *path, - int level, int slot); -enum btrfs_tree_block_status -btrfs_check_node(struct btrfs_fs_info *fs_info, - struct btrfs_key *parent_key, struct extent_buffer *buf); -enum btrfs_tree_block_status -btrfs_check_leaf(struct btrfs_fs_info *fs_info, - struct btrfs_key *parent_key, struct extent_buffer *buf); -void reada_for_search(struct btrfs_fs_info *fs_info, struct btrfs_path *path, - int level, int slot, u64 objectid); -struct extent_buffer *read_node_slot(struct btrfs_fs_info *fs_info, - struct extent_buffer *parent, int slot); -int btrfs_previous_item(struct btrfs_root *root, - struct btrfs_path *path, u64 min_objectid, - int type); -int btrfs_previous_extent_item(struct btrfs_root *root, - struct btrfs_path *path, u64 min_objectid); -int btrfs_next_extent_item(struct btrfs_root *root, - struct btrfs_path *path, u64 max_objectid); -int btrfs_cow_block(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct extent_buffer *buf, - struct extent_buffer *parent, int parent_slot, - struct extent_buffer **cow_ret); -int __btrfs_cow_block(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct extent_buffer *buf, - struct extent_buffer *parent, int parent_slot, - struct extent_buffer **cow_ret, - u64 search_start, u64 empty_size); -int btrfs_copy_root(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct extent_buffer *buf, - struct extent_buffer **cow_ret, u64 new_root_objectid); -int btrfs_create_root(struct btrfs_trans_handle *trans, - struct btrfs_fs_info *fs_info, u64 objectid); -int btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path, - u32 data_size); -int btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end); -int btrfs_split_item(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, - struct btrfs_key *new_key, - unsigned long split_offset); -int btrfs_search_slot(struct btrfs_trans_handle *trans, - struct btrfs_root *root, const struct btrfs_key *key, - struct btrfs_path *p, int ins_len, int cow); -int btrfs_search_slot_for_read(struct btrfs_root *root, - const struct btrfs_key *key, - struct btrfs_path *p, int find_higher, - int return_any); -int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, - int *slot); -int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *found_path, - u64 iobjectid, u64 ioff, u8 key_type, - struct btrfs_key *found_key); -void btrfs_release_path(struct btrfs_path *p); -void add_root_to_dirty_list(struct btrfs_root *root); -struct btrfs_path *btrfs_alloc_path(void); -void btrfs_free_path(struct btrfs_path *p); -void btrfs_init_path(struct btrfs_path *p); int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, int slot, int nr); @@ -2838,183 +2695,4 @@ static inline int btrfs_next_leaf(struct btrfs_root *root, return btrfs_next_sibling_tree_block(root->fs_info, path); } -static inline int btrfs_next_item(struct btrfs_root *root, - struct btrfs_path *p) -{ - ++p->slots[0]; - if (p->slots[0] >= btrfs_header_nritems(p->nodes[0])) { - int ret; - ret = btrfs_next_leaf(root, p); - /* - * Revert the increased slot, or the path may point to - * an invalid item. - */ - if (ret) - p->slots[0]--; - return ret; - } - return 0; -} - -int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path); -int btrfs_leaf_free_space(struct extent_buffer *leaf); -void btrfs_fixup_low_keys(struct btrfs_path *path, struct btrfs_disk_key *key, - int level); -int btrfs_set_item_key_safe(struct btrfs_root *root, struct btrfs_path *path, - struct btrfs_key *new_key); -void btrfs_set_item_key_unsafe(struct btrfs_root *root, - struct btrfs_path *path, - struct btrfs_key *new_key); - -u16 btrfs_super_csum_size(const struct btrfs_super_block *s); -const char *btrfs_super_csum_name(u16 csum_type); -u16 btrfs_csum_type_size(u16 csum_type); -size_t btrfs_super_num_csums(void); - -/* root-item.c */ -int btrfs_add_root_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *tree_root, - u64 root_id, u8 type, u64 ref_id, - u64 dirid, u64 sequence, - const char *name, int name_len); -int btrfs_insert_root(struct btrfs_trans_handle *trans, struct btrfs_root - *root, struct btrfs_key *key, struct btrfs_root_item - *item); -int btrfs_del_root(struct btrfs_trans_handle *trans, struct btrfs_root *root, - struct btrfs_key *key); -int btrfs_update_root(struct btrfs_trans_handle *trans, struct btrfs_root - *root, struct btrfs_key *key, struct btrfs_root_item - *item); -int btrfs_find_last_root(struct btrfs_root *root, u64 objectid, struct - btrfs_root_item *item, struct btrfs_key *key); -/* dir-item.c */ -int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, struct btrfs_root - *root, const char *name, int name_len, u64 dir, - struct btrfs_key *location, u8 type, u64 index); -struct btrfs_dir_item *btrfs_lookup_dir_item(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, u64 dir, - const char *name, int name_len, - int mod); -struct btrfs_dir_item *btrfs_lookup_dir_index_item(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, u64 dir, - u64 objectid, const char *name, int name_len, - int mod); -int btrfs_delete_one_dir_name(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, - struct btrfs_dir_item *di); -int btrfs_insert_xattr_item(struct btrfs_trans_handle *trans, - struct btrfs_root *root, const char *name, - u16 name_len, const void *data, u16 data_len, - u64 dir); -struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_root *root, - struct btrfs_path *path, - const char *name, int name_len); - -/* inode-item.c */ -int btrfs_insert_inode_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - const char *name, int name_len, - u64 inode_objectid, u64 ref_objectid, u64 index); -int btrfs_insert_inode(struct btrfs_trans_handle *trans, struct btrfs_root - *root, u64 objectid, struct btrfs_inode_item - *inode_item); -int btrfs_lookup_inode(struct btrfs_trans_handle *trans, struct btrfs_root - *root, struct btrfs_path *path, - struct btrfs_key *location, int mod); -struct btrfs_inode_extref *btrfs_lookup_inode_extref(struct btrfs_trans_handle - *trans, struct btrfs_path *path, struct btrfs_root *root, - u64 ino, u64 parent_ino, u64 index, const char *name, - int namelen, int ins_len); -int btrfs_del_inode_extref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - const char *name, int name_len, - u64 inode_objectid, u64 ref_objectid, - u64 *index); -int btrfs_insert_inode_extref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - const char *name, int name_len, - u64 inode_objectid, u64 ref_objectid, u64 index); -struct btrfs_inode_ref *btrfs_lookup_inode_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct btrfs_path *path, - const char *name, int namelen, u64 ino, u64 parent_ino, - int ins_len); -int btrfs_del_inode_ref(struct btrfs_trans_handle *trans, - struct btrfs_root *root, const char *name, int name_len, - u64 ino, u64 parent_ino, u64 *index); - -/* file-item.c */ -int btrfs_del_csums(struct btrfs_trans_handle *trans, u64 bytenr, u64 len); -int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 objectid, u64 pos, u64 offset, - u64 disk_num_bytes, - u64 num_bytes); -int btrfs_insert_inline_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, u64 objectid, - u64 offset, const char *buffer, size_t size); -int btrfs_csum_file_block(struct btrfs_trans_handle *trans, u64 alloc_end, - u64 bytenr, char *data, size_t len); - -/* uuid-tree.c, interface for mounted mounted filesystem */ -int btrfs_lookup_uuid_subvol_item(int fd, const u8 *uuid, u64 *subvol_id); -int btrfs_lookup_uuid_received_subvol_item(int fd, const u8 *uuid, - u64 *subvol_id); - -/* uuid-tree.c, interface for unmounte filesystem */ -int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, u8 *uuid, u8 type, - u64 subvol_id_cpu); -int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, u8 *uuid, u8 type, - u64 subid); - -static inline int is_fstree(u64 rootid) -{ - if (rootid == BTRFS_FS_TREE_OBJECTID || - (signed long long)rootid >= (signed long long)BTRFS_FIRST_FREE_OBJECTID) - return 1; - return 0; -} - -void btrfs_uuid_to_key(const u8 *uuid, struct btrfs_key *key); - -/* inode.c */ -int check_dir_conflict(struct btrfs_root *root, char *name, int namelen, - u64 dir, u64 index); -int btrfs_new_inode(struct btrfs_trans_handle *trans, struct btrfs_root *root, - u64 ino, u32 mode); -int btrfs_change_inode_flags(struct btrfs_trans_handle *trans, - struct btrfs_root *root, u64 ino, u64 flags); -int btrfs_add_link(struct btrfs_trans_handle *trans, struct btrfs_root *root, - u64 ino, u64 parent_ino, char *name, int namelen, - u8 type, u64 *index, int add_backref, int ignore_existed); -int btrfs_unlink(struct btrfs_trans_handle *trans, struct btrfs_root *root, - u64 ino, u64 parent_ino, u64 index, const char *name, - int namelen, int add_orphan); -int btrfs_add_orphan_item(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct btrfs_path *path, - u64 ino); -int btrfs_mkdir(struct btrfs_trans_handle *trans, struct btrfs_root *root, - char *name, int namelen, u64 parent_ino, u64 *ino, int mode); -struct btrfs_root *btrfs_mksubvol(struct btrfs_root *root, const char *base, - u64 root_objectid, bool convert); -int btrfs_find_free_objectid(struct btrfs_trans_handle *trans, - struct btrfs_root *fs_root, - u64 dirid, u64 *objectid); - -/* file.c */ -int btrfs_get_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, - u64 ino, u64 offset, u64 len, int ins_len); -int btrfs_punch_hole(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 ino, u64 offset, u64 len); -int btrfs_read_file(struct btrfs_root *root, u64 ino, u64 start, int len, - char *dest); - -/* extent-tree.c */ -int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, unsigned long nr); - #endif diff --git a/mkfs/main.c b/mkfs/main.c index 4e0a46a77a..ce096d3621 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1021,7 +1021,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv) while(1) { int c; enum { - GETOPT_VAL_SHRINK = 257, + GETOPT_VAL_SHRINK = GETOPT_VAL_FIRST, GETOPT_VAL_CHECKSUM, GETOPT_VAL_GLOBAL_ROOTS, }; diff --git a/tests/misc-tests/052-receive-write-encoded/test.sh b/tests/misc-tests/052-receive-write-encoded/test.sh new file mode 100755 index 0000000000..47330281e5 --- /dev/null +++ b/tests/misc-tests/052-receive-write-encoded/test.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# test that we can send and receive encoded writes for three modes of +# transparent compression: zlib, lzo, and zstd. + +source "$TEST_TOP/common" + +check_prereq mkfs.btrfs +check_prereq btrfs + +setup_root_helper +prepare_test_dev + +here=`pwd` + +# assumes the filesystem exists, and does mount, write, snapshot, send, unmount +# for the specified encoding option +send_one() { + local str + local subv + local snap + + algorithm="$1" + shift + str="$1" + shift + + subv="subv-$algorithm" + snap="snap-$algorithm" + + run_check_mount_test_dev "-o" "compress-force=$algorithm" + cd "$TEST_MNT" || _fail "cannot chdir to TEST_MNT" + + run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$subv" + run_check $SUDO_HELPER dd if=/dev/zero of="$subv/file1" bs=1M count=1 + run_check $SUDO_HELPER dd if=/dev/zero of="$subv/file2" bs=500K count=1 + run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$subv" "$snap" + run_check $SUDO_HELPER "$TOP/btrfs" send -f "$str" "$snap" "$@" + + cd "$here" || _fail "cannot chdir back to test directory" + run_check_umount_test_dev +} + +receive_one() { + local str + str="$1" + shift + + run_check_mkfs_test_dev + run_check_mount_test_dev + run_check $SUDO_HELPER "$TOP/btrfs" receive "$@" -v -f "$str" "$TEST_MNT" + run_check_umount_test_dev + run_check rm -f -- "$str" +} + +test_one_write_encoded() { + local str + local algorithm + algorithm="$1" + shift + str="$here/stream-$algorithm.stream" + + run_check_mkfs_test_dev + send_one "$algorithm" "$str" --compressed-data + receive_one "$str" "$@" +} + +test_one_stream_v1() { + local str + local algorithm + algorithm="$1" + shift + str="$here/stream-$algorithm.stream" + + run_check_mkfs_test_dev + send_one "$algorithm" "$str" --proto 1 + receive_one "$str" "$@" +} + +test_mix_write_encoded() { + local strzlib + local strlzo + local strzstd + strzlib="$here/stream-zlib.stream" + strlzo="$here/stream-lzo.stream" + strzstd="$here/stream-zstd.stream" + + run_check_mkfs_test_dev + + send_one "zlib" "$strzlib" --compressed-data + send_one "lzo" "$strlzo" --compressed-data + send_one "zstd" "$strzstd" --compressed-data + + receive_one "$strzlib" + receive_one "$strlzo" + receive_one "$strzstd" +} + +test_one_write_encoded "zlib" +test_one_write_encoded "lzo" +test_one_write_encoded "zstd" + +# with decompression forced +test_one_write_encoded "zlib" "--force-decompress" +test_one_write_encoded "lzo" "--force-decompress" +test_one_write_encoded "zstd" "--force-decompress" + +# send stream v1 +test_one_stream_v1 "zlib" +test_one_stream_v1 "lzo" +test_one_stream_v1 "zstd" + +# files use a mix of compression algorithms +test_mix_write_encoded