diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 00:58:56 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 00:58:56 +0000 |
commit | 8f4e216153a7178309882cd60df8162d97c2f23c (patch) | |
tree | 25a37c478eabdeaabeb7e5c38e6558dc272cc362 | |
parent | 4dd9079df53471e7b48fe6442696270a8fa26a56 (diff) | |
parent | 3888d132f2cfbf66afa3149706c77a48535c80fc (diff) | |
download | erofs-utils-android14-mainline-wifi-release.tar.gz |
Snap for 10447354 from 3888d132f2cfbf66afa3149706c77a48535c80fc to mainline-wifi-releaseaml_wif_341711020aml_wif_341610000aml_wif_341510000aml_wif_341410080aml_wif_341310010aml_wif_341110010aml_wif_341011010aml_wif_340913010android14-mainline-wifi-release
Change-Id: I76adff3351a1be5eb75c4563427704d55d49b448
63 files changed, 1565 insertions, 796 deletions
@@ -1,6 +1,7 @@ EROFS USERSPACE UTILITIES M: Li Guifu <bluce.lee@aliyun.com> M: Gao Xiang <xiang@kernel.org> +M: Huang Jianan <huangjianan@oppo.com> R: Chao Yu <chao@kernel.org> R: Miao Xie <miaoxie@huawei.com> R: Fang Wei <fangwei1@huawei.com> @@ -57,6 +57,7 @@ cc_defaults { cflags: [ "-Wall", "-Werror", + "-Wno-error=#warnings", "-Wno-ignored-qualifiers", "-Wno-pointer-arith", "-Wno-unused-parameter", @@ -68,6 +69,10 @@ cc_defaults { "-DLZ4_ENABLED", "-DLZ4HC_ENABLED", "-DWITH_ANDROID", + "-DHAVE_MEMRCHR", + "-DHAVE_SYS_IOCTL_H", + "-DHAVE_LLISTXATTR", + "-DHAVE_LGETXATTR", ], } @@ -144,6 +149,7 @@ cc_binary_host { name: "make_erofs", defaults: ["mkfs-erofs_defaults"], + stl: "libc++_static" } cc_binary { @@ -1,359 +1,15 @@ -Valid-License-Identifier: GPL-2.0 -Valid-License-Identifier: GPL-2.0-only -Valid-License-Identifier: GPL-2.0+ -Valid-License-Identifier: GPL-2.0-or-later -SPDX-URL: https://spdx.org/licenses/GPL-2.0.html -Usage-Guide: - To use this license in source code, put one of the following SPDX - tag/value pairs into a comment according to the placement - guidelines in the licensing rules documentation. - For 'GNU General Public License (GPL) version 2 only' use: - SPDX-License-Identifier: GPL-2.0 - or - SPDX-License-Identifier: GPL-2.0-only - For 'GNU General Public License (GPL) version 2 or any later version' use: - SPDX-License-Identifier: GPL-2.0+ - or - SPDX-License-Identifier: GPL-2.0-or-later -License-Text: +erofs-utils uses two different license patterns: - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + - most liberofs files in `lib` and `include` directories + use GPL-2.0+ OR Apache-2.0 dual license; - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. + - all other files use GPL-2.0+ license, unless + explicitly stated otherwise. - Preamble +Relevant licenses can be found in the LICENSES directory. - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. +This model is selected to emphasize that +files in `lib` and `include` directory are designed to be included into +3rd-party applications, while all other files, are intended to be used +"as is", as part of their intended scenarios, with no intention to +support 3rd-party integration use cases. @@ -1,3 +1,16 @@ +erofs-utils 1.5 + + * This release includes the following updates: + - (fsck.erofs) support filesystem extraction (Igor Ostapenko); + - support ztailpacking inline feature for compressed files (Yue Hu); + - (dump.erofs) support listing directories; + - more liberofs APIs (including iterate APIs) (me, Kelvin Zhang); + - use mtime to allow more control over the timestamps (David Anderson); + - switch to GPL-2.0+ OR Apache-2.0 dual license for liberofs; + - various bugfixes and cleanups; + + -- Gao Xiang <xiang@kernel.org> Mon, 13 Jun 2022 00:00:00 +0800 + erofs-utils 1.4 * This release includes the following updates: diff --git a/LICENSES/Apache-2.0 b/LICENSES/Apache-2.0 new file mode 100644 index 0000000..f6c1877 --- /dev/null +++ b/LICENSES/Apache-2.0 @@ -0,0 +1,186 @@ +Valid-License-Identifier: Apache-2.0 +SPDX-URL: https://spdx.org/licenses/Apache-2.0.html +Usage-Guide: + The Apache-2.0 may only be used for dual-licensed files where the other + license is GPL2 compatible. If you end up using this it MUST be used + together with a GPL2 compatible license using "OR". + To use the Apache License version 2.0 put the following SPDX tag/value + pair into a comment according to the placement guidelines in the + licensing rules documentation: + SPDX-License-Identifier: Apache-2.0 +License-Text: + +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the +copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other +entities that control, are controlled by, or are under common control with +that entity. For the purposes of this definition, "control" means (i) the +power, direct or indirect, to cause the direction or management of such +entity, whether by contract or otherwise, or (ii) ownership of fifty +percent (50%) or more of the outstanding shares, or (iii) beneficial +ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation source, +and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation +or translation of a Source form, including but not limited to compiled +object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, +made available under the License, as indicated by a copyright notice that +is included in or attached to the work (an example is provided in the +Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, +that is based on (or derived from) the Work and for which the editorial +revisions, annotations, elaborations, or other modifications represent, as +a whole, an original work of authorship. For the purposes of this License, +Derivative Works shall not include works that remain separable from, or +merely link (or bind by name) to the interfaces of, the Work and Derivative +Works thereof. + +"Contribution" shall mean any work of authorship, including the original +version of the Work and any modifications or additions to that Work or +Derivative Works thereof, that is intentionally submitted to Licensor for +inclusion in the Work by the copyright owner or by an individual or Legal +Entity authorized to submit on behalf of the copyright owner. For the +purposes of this definition, "submitted" means any form of electronic, +verbal, or written communication sent to the Licensor or its +representatives, including but not limited to communication on electronic +mailing lists, source code control systems, and issue tracking systems that +are managed by, or on behalf of, the Licensor for the purpose of discussing +and improving the Work, but excluding communication that is conspicuously +marked or otherwise designated in writing by the copyright owner as "Not a +Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on +behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable copyright license to + reproduce, prepare Derivative Works of, publicly display, publicly + perform, sublicense, and distribute the Work and such Derivative Works + in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable (except as stated in + this section) patent license to make, have made, use, offer to sell, + sell, import, and otherwise transfer the Work, where such license + applies only to those patent claims licensable by such Contributor that + are necessarily infringed by their Contribution(s) alone or by + combination of their Contribution(s) with the Work to which such + Contribution(s) was submitted. If You institute patent litigation + against any entity (including a cross-claim or counterclaim in a + lawsuit) alleging that the Work or a Contribution incorporated within + the Work constitutes direct or contributory patent infringement, then + any patent licenses granted to You under this License for that Work + shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or + Derivative Works thereof in any medium, with or without modifications, + and in Source or Object form, provided that You meet the following + conditions: + + a. You must give any other recipients of the Work or Derivative Works a + copy of this License; and + + b. You must cause any modified files to carry prominent notices stating + that You changed the files; and + + c. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices + from the Source form of the Work, excluding those notices that do not + pertain to any part of the Derivative Works; and + + d. If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained within + such NOTICE file, excluding those notices that do not pertain to any + part of the Derivative Works, in at least one of the following + places: within a NOTICE text file distributed as part of the + Derivative Works; within the Source form or documentation, if + provided along with the Derivative Works; or, within a display + generated by the Derivative Works, if and wherever such third-party + notices normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. You may + add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the + Work, provided that such additional attribution notices cannot be + construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions stated + in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any + Contribution intentionally submitted for inclusion in the Work by You to + the Licensor shall be under the terms and conditions of this License, + without any additional terms or conditions. Notwithstanding the above, + nothing herein shall supersede or modify the terms of any separate + license agreement you may have executed with Licensor regarding such + Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to + in writing, Licensor provides the Work (and each Contributor provides + its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied, including, without limitation, + any warranties or conditions of TITLE, NON-INFRINGEMENT, + MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely + responsible for determining the appropriateness of using or + redistributing the Work and assume any risks associated with Your + exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether + in tort (including negligence), contract, or otherwise, unless required + by applicable law (such as deliberate and grossly negligent acts) or + agreed to in writing, shall any Contributor be liable to You for + damages, including any direct, indirect, special, incidental, or + consequential damages of any character arising as a result of this + License or out of the use or inability to use the Work (including but + not limited to damages for loss of goodwill, work stoppage, computer + failure or malfunction, or any and all other commercial damages or + losses), even if such Contributor has been advised of the possibility of + such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the + Work or Derivative Works thereof, You may choose to offer, and charge a + fee for, acceptance of support, warranty, indemnity, or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, such + Contributor by reason of your accepting any such warranty or additional + liability. + +END OF TERMS AND CONDITIONS diff --git a/LICENSES/GPL-2.0 b/LICENSES/GPL-2.0 new file mode 100644 index 0000000..b7eaaf4 --- /dev/null +++ b/LICENSES/GPL-2.0 @@ -0,0 +1,359 @@ +Valid-License-Identifier: GPL-2.0 +Valid-License-Identifier: GPL-2.0-only +Valid-License-Identifier: GPL-2.0+ +Valid-License-Identifier: GPL-2.0-or-later +SPDX-URL: https://spdx.org/licenses/GPL-2.0.html +Usage-Guide: + To use this license in source code, put one of the following SPDX + tag/value pairs into a comment according to the placement + guidelines in the licensing rules documentation. + For 'GNU General Public License (GPL) version 2 only' use: + SPDX-License-Identifier: GPL-2.0 + or + SPDX-License-Identifier: GPL-2.0-only + For 'GNU General Public License (GPL) version 2 or any later version' use: + SPDX-License-Identifier: GPL-2.0+ + or + SPDX-License-Identifier: GPL-2.0-or-later +License-Text: + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update erofs-utils +# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md + name: "erofs-utils" description: "EROFS Utilities" third_party { @@ -5,11 +9,11 @@ third_party { type: GIT value: "https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git" } - version: "1.4" + version: "v1.5" license_type: RESTRICTED last_upgrade_date { - year: 2021 - month: 11 - day: 22 + year: 2023 + month: 1 + day: 18 } } @@ -1,9 +1,13 @@ erofs-utils =========== -erofs-utils includes user-space tools for EROFS filesystem. -Currently mkfs.erofs, (experimental) erofsfuse, dump.erofs, fsck.erofs -are available. +userspace tools for EROFS filesystem, currently including: + + mkfs.erofs filesystem formatter + erofsfuse FUSE daemon alternative + dump.erofs filesystem analyzer + fsck.erofs filesystem compatibility & consistency checker as well + as extractor Dependencies & build -------------------- @@ -59,6 +63,7 @@ In order to enable LZMA support, build with the following commands: Additionally, you could specify liblzma build paths with: --with-liblzma-incdir and --with-liblzma-libdir + mkfs.erofs ---------- @@ -133,8 +138,9 @@ git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git -b obsoleted PLEASE NOTE: This version is highly _NOT recommended_ now. -erofsfuse (experimental) ------------------------- + +erofsfuse +--------- erofsfuse is introduced to support EROFS format for various platforms (including older linux kernels) and new on-disk features iteration. @@ -147,9 +153,9 @@ significant I/O overhead, double caching, etc.) Therefore, NEVER use it if performance is the top concern. -Note that xattr & ACL aren't implemented yet due to the current Android -use-case vs limited time. If you have some interest, contribution is, -as always, welcome. +Note that extended attributes and ACLs aren't implemented yet due to +the current Android use case vs limited time. If you are interested, +contribution is, as always, welcome. How to build erofsfuse ~~~~~~~~~~~~~~~~~~~~~~ @@ -178,24 +184,52 @@ To debug erofsfuse (also automatically run in foreground): To unmount an erofsfuse mountpoint as a non-root user: $ fusermount -u foo/ -dump.erofs and fsck.erofs (experimental) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -dump.erofs and fsck.erofs are two new experimental tools to analyse -and check EROFS file systems. +dump.erofs and fsck.erofs +------------------------- + +dump.erofs and fsck.erofs are used to analyze, check, and extract +EROFS filesystems. Note that extended attributes and ACLs are still +unsupported when extracting images with fsck.erofs. + +Container images +---------------- + +EROFS filesystem is well-suitably used for container images with +advanced features like chunk-based files, multi-devices (blobs) +and new fscache backend for lazy pulling and cache management, etc. + +For example, CNCF Dragonfly Nydus image service [7] introduces an +(EROFS-compatible) RAFS v6 image format to overcome flaws of the +current OCIv1 tgz images so that: -They are still incomplete and actively under development by the -community. But you could check them out if needed in advance. + - Images can be downloaded on demand in chunks aka lazy pulling with + new fscache backend (5.19+) or userspace block devices (5.16+); + + - Finer chunk-based content-addressable data deduplication to minimize + storage, transmission and memory footprints; + + - Merged filesystem tree to remove all metadata of intermediate layers + as an option; + + - (e)stargz, zstd::chunked and other formats can be converted and run + on the fly; + + - and more. + +Apart from Dragonfly Nydus, a native user daemon is planned to be added +to erofs-utils to parse EROFS, (e)stargz and zstd::chunked images from +network too as a real part of EROFS filesystem project. -Report, feedback and/or contribution are welcomed. Contribution ------------ -erofs-utils is under GPLv2+ as a part of EROFS filesystem project, -feel free to send patches or feedback to: +erofs-utils is a part of EROFS filesystem project, feel free to send +patches or feedback to: linux-erofs mailing list <linux-erofs@lists.ozlabs.org> + Comments -------- @@ -251,3 +285,6 @@ Comments which is also resolved in lz4-1.9.3. [6] https://tukaani.org/xz/xz-5.3.2alpha.tar.xz + +[7] https://nydus.dev + https://github.com/dragonflyoss/image-service @@ -1,2 +1,2 @@ -1.4 -2021-11-22 +1.5 +2022-06-13 diff --git a/configure.ac b/configure.ac index 6f7e271..a736ff0 100644 --- a/configure.ac +++ b/configure.ac @@ -11,7 +11,7 @@ AC_CONFIG_SRCDIR([config.h.in]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR(config) -AM_INIT_AUTOMAKE([foreign -Wall -Werror]) +AM_INIT_AUTOMAKE([foreign -Wall]) # Checks for programs. AM_PROG_AR @@ -65,6 +65,12 @@ AC_ARG_ENABLE([debug], [enable_debug="$enableval"], [enable_debug="no"]) +AC_ARG_ENABLE([werror], + [AS_HELP_STRING([--enable-werror], + [enable -Werror @<:@default=no@:>@])], + [enable_werror="$enableval"], + [enable_werror="no"]) + AC_ARG_ENABLE(lz4, [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])], [enable_lz4="$enableval"], [enable_lz4="yes"]) @@ -134,6 +140,8 @@ AC_CHECK_HEADERS(m4_flatten([ unistd.h ])) +AC_HEADER_TIOCGWINSZ + # Checks for typedefs, structures, and compiler characteristics. AC_C_INLINE AC_TYPE_INT64_T @@ -167,6 +175,11 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1, #define _LARGEFILE64_SOURCE #include <unistd.h>]) +AC_CHECK_DECL(memrchr,[AC_DEFINE(HAVE_MEMRCHR, 1, + [Define to 1 if memrchr declared in string.h])],, + [#define _GNU_SOURCE + #include <string.h>]) + # Checks for library functions. AC_CHECK_FUNCS(m4_flatten([ backtrace @@ -192,6 +205,11 @@ AS_IF([test "x$enable_debug" != "xno"], [], [ CPPFLAGS="$CPPFLAGS -DNDEBUG" ]) +# Configure -Werror +AS_IF([test "x$enable_werror" != "xyes"], [], [ + CPPFLAGS="$CPPFLAGS -Werror" +]) + # Configure libuuid AS_IF([test "x$with_uuid" != "xno"], [ PKG_CHECK_MODULES([libuuid], [uuid]) diff --git a/dump/Makefile.am b/dump/Makefile.am index 9f0cd3f..c2bef6d 100644 --- a/dump/Makefile.am +++ b/dump/Makefile.am @@ -5,6 +5,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = dump.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} dump_erofs_SOURCES = main.c -dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +dump_erofs_CFLAGS = -Wall -I$(top_srcdir)/include dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} diff --git a/dump/main.c b/dump/main.c index 72761bd..49ff2b7 100644 --- a/dump/main.c +++ b/dump/main.c @@ -5,12 +5,16 @@ * Created by Wang Qi <mpiglet@outlook.com> * Guo Xuenan <guoxuenan@huawei.com> */ +#define _GNU_SOURCE #include <stdlib.h> #include <getopt.h> #include <time.h> +#include <sys/stat.h> #include "erofs/print.h" #include "erofs/inode.h" #include "erofs/io.h" +#include "erofs/dir.h" +#include "../lib/liberofs_private.h" #ifdef HAVE_LIBUUID #include <uuid.h> @@ -22,6 +26,7 @@ struct erofsdump_cfg { bool show_extent; bool show_superblock; bool show_statistics; + bool show_subdirectories; erofs_nid_t nid; const char *inode_path; }; @@ -73,6 +78,7 @@ static struct option long_options[] = { {"nid", required_argument, NULL, 2}, {"device", required_argument, NULL, 3}, {"path", required_argument, NULL, 4}, + {"ls", no_argument, NULL, 5}, {0, 0, 0, 0}, }; @@ -91,10 +97,7 @@ static struct erofsdump_feature feature_lists[] = { { false, EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, "device_table" }, }; -static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid); -static inline int erofs_checkdirent(struct erofs_dirent *de, - struct erofs_dirent *last_de, - u32 maxsize, const char *dname); +static int erofsdump_readdir(struct erofs_dir_context *ctx); static void usage(void) { @@ -102,9 +105,10 @@ static void usage(void) "Dump erofs layout from IMAGE, and [options] are:\n" " -S show statistic information of the image\n" " -V print the version number of dump.erofs and exit.\n" - " -e show extent info (--nid is required)\n" + " -e show extent info (INODE required)\n" " -s show information about superblock\n" " --device=X specify an extra device to be used together\n" + " --ls show directory contents (INODE required)\n" " --nid=# show the target inode info of nid #\n" " --path=X show the target inode info of path X\n" " --help display this help and exit.\n", @@ -113,7 +117,7 @@ static void usage(void) static void erofsdump_print_version(void) { - fprintf(stderr, "dump.erofs %s\n", cfg.c_version); + printf("dump.erofs %s\n", cfg.c_version); } static int erofsdump_parse_options_cfg(int argc, char **argv) @@ -157,6 +161,9 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) dumpcfg.show_inode = true; ++dumpcfg.totalshow; break; + case 5: + dumpcfg.show_subdirectories = true; + break; default: return -EINVAL; } @@ -178,7 +185,7 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) return 0; } -static int erofs_get_occupied_size(struct erofs_inode *inode, +static int erofsdump_get_occupied_size(struct erofs_inode *inode, erofs_off_t *size) { *size = 0; @@ -196,23 +203,25 @@ static int erofs_get_occupied_size(struct erofs_inode *inode, break; default: erofs_err("unknown datalayout"); - return -1; + return -ENOTSUP; } return 0; } -static int erofs_getfile_extension(const char *filename) +static void inc_file_extension_count(const char *dname, unsigned int len) { - char *postfix = strrchr(filename, '.'); - int type = 0; + char *postfix = memrchr(dname, '.', len); + int type; - if (!postfix) - return OTHERFILETYPE - 1; - for (type = 0; type < OTHERFILETYPE - 1; ++type) { - if (strcmp(postfix, file_types[type]) == 0) - break; + if (!postfix) { + type = OTHERFILETYPE - 1; + } else { + for (type = 0; type < OTHERFILETYPE - 1; ++type) + if (!strncmp(postfix, file_types[type], + len - (postfix - dname))) + break; } - return type; + ++stats.file_type_stat[type]; } static void update_file_size_statatics(erofs_off_t occupied_size, @@ -247,192 +256,67 @@ static void update_file_size_statatics(erofs_off_t occupied_size, stats.file_comp_size[occupied_size_mark]++; } -static inline int erofs_checkdirent(struct erofs_dirent *de, - struct erofs_dirent *last_de, - u32 maxsize, const char *dname) +static int erofsdump_ls_dirent_iter(struct erofs_dir_context *ctx) { - int dname_len; - unsigned int nameoff = le16_to_cpu(de->nameoff); - - if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { - erofs_err("invalid de[0].nameoff %u @ nid %llu", - nameoff, de->nid | 0ULL); - return -EFSCORRUPTED; - } + char fname[EROFS_NAME_LEN + 1]; - dname_len = (de + 1 >= last_de) ? strnlen(dname, maxsize - nameoff) : - le16_to_cpu(de[1].nameoff) - nameoff; - /* a corrupted entry is found */ - if (nameoff + dname_len > maxsize || - dname_len > EROFS_NAME_LEN) { - erofs_err("bogus dirent @ nid %llu", - le64_to_cpu(de->nid) | 0ULL); - DBG_BUGON(1); - return -EFSCORRUPTED; - } - if (de->file_type >= EROFS_FT_MAX) { - erofs_err("invalid file type %llu", de->nid); - return -EFSCORRUPTED; - } - return dname_len; + strncpy(fname, ctx->dname, ctx->de_namelen); + fname[ctx->de_namelen] = '\0'; + fprintf(stdout, "%10llu %u %s\n", ctx->de_nid | 0ULL, + ctx->de_ftype, fname); + return 0; } -static int erofs_read_dirent(struct erofs_dirent *de, - erofs_nid_t nid, erofs_nid_t parent_nid, - const char *dname) +static int erofsdump_dirent_iter(struct erofs_dir_context *ctx) +{ + /* skip "." and ".." dentry */ + if (ctx->dot_dotdot) + return 0; + + return erofsdump_readdir(ctx); +} + +static int erofsdump_readdir(struct erofs_dir_context *ctx) { int err; erofs_off_t occupied_size = 0; - struct erofs_inode inode = { .nid = de->nid }; + struct erofs_inode vi = { .nid = ctx->de_nid }; - stats.files++; - stats.file_category_stat[de->file_type]++; - err = erofs_read_inode_from_disk(&inode); + err = erofs_read_inode_from_disk(&vi); if (err) { - erofs_err("read file inode from disk failed!"); + erofs_err("failed to read file inode from disk"); return err; } + stats.files++; + stats.file_category_stat[erofs_mode_to_ftype(vi.i_mode)]++; - err = erofs_get_occupied_size(&inode, &occupied_size); + err = erofsdump_get_occupied_size(&vi, &occupied_size); if (err) { - erofs_err("get file size failed\n"); + erofs_err("get file size failed"); return err; } - if (de->file_type == EROFS_FT_REG_FILE) { - stats.files_total_origin_size += inode.i_size; - stats.file_type_stat[erofs_getfile_extension(dname)]++; + if (S_ISREG(vi.i_mode)) { + stats.files_total_origin_size += vi.i_size; + inc_file_extension_count(ctx->dname, ctx->de_namelen); stats.files_total_size += occupied_size; - update_file_size_statatics(occupied_size, inode.i_size); - } - - if ((de->file_type == EROFS_FT_DIR) - && de->nid != nid && de->nid != parent_nid) { - err = erofs_read_dir(de->nid, nid); - if (err) { - erofs_err("parse dir nid %llu error occurred\n", - de->nid); - return err; - } + update_file_size_statatics(occupied_size, vi.i_size); } - return 0; -} - -static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid) -{ - int err; - erofs_off_t offset; - char buf[EROFS_BLKSIZ]; - struct erofs_inode vi = { .nid = nid }; - err = erofs_read_inode_from_disk(&vi); - if (err) - return err; + /* XXXX: the dir depth should be restricted in order to avoid loops */ + if (S_ISDIR(vi.i_mode)) { + struct erofs_dir_context nctx = { + .flags = ctx->dir ? EROFS_READDIR_VALID_PNID : 0, + .pnid = ctx->dir ? ctx->dir->nid : 0, + .dir = &vi, + .cb = erofsdump_dirent_iter, + }; - offset = 0; - while (offset < vi.i_size) { - erofs_off_t maxsize = min_t(erofs_off_t, - vi.i_size - offset, EROFS_BLKSIZ); - struct erofs_dirent *de = (void *)buf; - struct erofs_dirent *end; - unsigned int nameoff; - - err = erofs_pread(&vi, buf, maxsize, offset); - if (err) - return err; - - nameoff = le16_to_cpu(de->nameoff); - end = (void *)buf + nameoff; - while (de < end) { - const char *dname; - int ret; - - /* skip "." and ".." dentry */ - if (de->nid == nid || de->nid == parent_nid) { - de++; - continue; - } - - dname = (char *)buf + nameoff; - ret = erofs_checkdirent(de, end, maxsize, dname); - if (ret < 0) - return ret; - ret = erofs_read_dirent(de, nid, parent_nid, dname); - if (ret < 0) - return ret; - ++de; - } - offset += maxsize; + return erofs_iterate_dir(&nctx, false); } return 0; } -static int erofs_get_pathname(erofs_nid_t nid, erofs_nid_t parent_nid, - erofs_nid_t target, char *path, unsigned int pos) -{ - int err; - erofs_off_t offset; - char buf[EROFS_BLKSIZ]; - struct erofs_inode inode = { .nid = nid }; - - path[pos++] = '/'; - if (target == sbi.root_nid) - return 0; - - err = erofs_read_inode_from_disk(&inode); - if (err) { - erofs_err("read inode failed @ nid %llu", nid | 0ULL); - return err; - } - - offset = 0; - while (offset < inode.i_size) { - erofs_off_t maxsize = min_t(erofs_off_t, - inode.i_size - offset, EROFS_BLKSIZ); - struct erofs_dirent *de = (void *)buf; - struct erofs_dirent *end; - unsigned int nameoff; - - err = erofs_pread(&inode, buf, maxsize, offset); - if (err) - return err; - - nameoff = le16_to_cpu(de->nameoff); - end = (void *)buf + nameoff; - while (de < end) { - const char *dname; - int len; - - nameoff = le16_to_cpu(de->nameoff); - dname = (char *)buf + nameoff; - len = erofs_checkdirent(de, end, maxsize, dname); - if (len < 0) - return len; - - if (de->nid == target) { - memcpy(path + pos, dname, len); - path[pos + len] = '\0'; - return 0; - } - - if (de->file_type == EROFS_FT_DIR && - de->nid != parent_nid && - de->nid != nid) { - memcpy(path + pos, dname, len); - err = erofs_get_pathname(de->nid, nid, - target, path, pos + len); - if (!err) - return 0; - memset(path + pos, 0, len); - } - ++de; - } - offset += maxsize; - } - return -1; -} - static int erofsdump_map_blocks(struct erofs_inode *inode, struct erofs_map_blocks *map, int flags) { @@ -451,7 +335,7 @@ static void erofsdump_show_fileinfo(bool show_extent) erofs_off_t size; u16 access_mode; struct erofs_inode inode = { .nid = dumpcfg.nid }; - char path[PATH_MAX + 1] = {0}; + char path[PATH_MAX]; char access_mode_str[] = "rwxrwxrwx"; char timebuf[128] = {0}; unsigned int extent_count = 0; @@ -481,8 +365,7 @@ static void erofsdump_show_fileinfo(bool show_extent) return; } - err = erofs_get_pathname(sbi.root_nid, sbi.root_nid, - inode.nid, path, 0); + err = erofs_get_pathname(inode.nid, path, sizeof(path)); if (err < 0) { erofs_err("file path not found @ nid %llu", inode.nid | 0ULL); return; @@ -510,10 +393,29 @@ static void erofsdump_show_fileinfo(bool show_extent) fprintf(stdout, "Access: %04o/%s\n", access_mode, access_mode_str); fprintf(stdout, "Timestamp: %s.%09d\n", timebuf, inode.i_mtime_nsec); + if (dumpcfg.show_subdirectories) { + struct erofs_dir_context ctx = { + .flags = EROFS_READDIR_VALID_PNID, + .pnid = inode.nid, + .dir = &inode, + .cb = erofsdump_ls_dirent_iter, + .de_nid = 0, + .dname = "", + .de_namelen = 0, + }; + + fprintf(stdout, "\n NID TYPE FILENAME\n"); + err = erofs_iterate_dir(&ctx, false); + if (err) { + erofs_err("failed to list directory contents"); + return; + } + } + if (!dumpcfg.show_extent) return; - fprintf(stdout, "\n Ext: logical offset | length : physical offset | length \n"); + fprintf(stdout, "\n Ext: logical offset | length : physical offset | length\n"); while (map.m_la < inode.i_size) { struct erofs_map_dev mdev; @@ -631,13 +533,21 @@ static void erofsdump_file_statistic(void) static void erofsdump_print_statistic(void) { int err; + struct erofs_dir_context ctx = { + .flags = 0, + .pnid = 0, + .dir = NULL, + .cb = erofsdump_dirent_iter, + .de_nid = sbi.root_nid, + .dname = "", + .de_namelen = 0, + }; - err = erofs_read_dir(sbi.root_nid, sbi.root_nid); + err = erofsdump_readdir(&ctx); if (err) { erofs_err("read dir failed"); return; } - erofsdump_file_statistic(); erofsdump_filesize_distribution("Original", stats.file_original_size, diff --git a/fsck/Makefile.am b/fsck/Makefile.am index 55b31ea..e6a1fb6 100644 --- a/fsck/Makefile.am +++ b/fsck/Makefile.am @@ -5,6 +5,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = fsck.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} fsck_erofs_SOURCES = main.c -fsck_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +fsck_erofs_CFLAGS = -Wall -I$(top_srcdir)/include fsck_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} diff --git a/fsck/main.c b/fsck/main.c index 0af15b4..5a2f659 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -89,7 +89,7 @@ static void usage(void) static void erofsfsck_print_version(void) { - fprintf(stderr, "fsck.erofs %s\n", cfg.c_version); + printf("fsck.erofs %s\n", cfg.c_version); } static int erofsfsck_parse_options_cfg(int argc, char **argv) @@ -638,6 +638,44 @@ out: return ret; } +static int erofs_extract_special(struct erofs_inode *inode) +{ + bool tryagain = true; + int ret; + + erofs_dbg("extract special to path: %s", fsckcfg.extract_path); + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, -1); + if (ret) + return ret; + +again: + if (mknod(fsckcfg.extract_path, inode->i_mode, inode->u.i_rdev) < 0) { + if (errno == EEXIST && fsckcfg.overwrite && tryagain) { + erofs_warn("try to forcely remove file %s", + fsckcfg.extract_path); + if (unlink(fsckcfg.extract_path) < 0) { + erofs_err("failed to remove: %s", + fsckcfg.extract_path); + return -errno; + } + tryagain = false; + goto again; + } + if (errno == EEXIST || fsckcfg.superuser) { + erofs_err("failed to create special file: %s", + fsckcfg.extract_path); + ret = -errno; + } else { + erofs_warn("failed to create special file: %s, skipped", + fsckcfg.extract_path); + ret = -ECANCELED; + } + } + return ret; +} + static int erofsfsck_dirent_iter(struct erofs_dir_context *ctx) { int ret; @@ -698,6 +736,12 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) case S_IFLNK: ret = erofs_extract_symlink(&inode); break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + ret = erofs_extract_special(&inode); + break; default: /* TODO */ goto verify; @@ -707,7 +751,7 @@ verify: /* verify data chunk layout */ ret = erofs_verify_inode_data(&inode, -1); } - if (ret) + if (ret && ret != -ECANCELED) goto out; /* XXXX: the dir depth should be restricted in order to avoid loops */ @@ -725,6 +769,8 @@ verify: if (!ret) erofsfsck_set_attributes(&inode, fsckcfg.extract_path); + if (ret == -ECANCELED) + ret = 0; out: if (ret && ret != -EIO) fsckcfg.corrupted = true; diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 0a78c0a..3179a2b 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-2.0+ AUTOMAKE_OPTIONS = foreign +noinst_HEADERS = $(top_srcdir)/fuse/macosx.h bin_PROGRAMS = erofsfuse erofsfuse_SOURCES = main.c -erofsfuse_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +erofsfuse_CFLAGS = -Wall -I$(top_srcdir)/include erofsfuse_CFLAGS += -DFUSE_USE_VERSION=26 ${libfuse_CFLAGS} ${libselinux_CFLAGS} erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse_LIBS} ${liblz4_LIBS} \ ${libselinux_LIBS} ${liblzma_LIBS} diff --git a/fuse/main.c b/fuse/main.c index ae377ae..f4c2476 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -61,7 +61,6 @@ int erofsfuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, #else return erofs_iterate_dir(&ctx.ctx, true); #endif - } static void *erofsfuse_init(struct fuse_conn_info *info) @@ -258,7 +257,7 @@ int main(int argc, char *argv[]) struct fuse_args args = FUSE_ARGS_INIT(argc, argv); erofs_init_configure(); - fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version); + printf("%s %s\n", basename(argv[0]), cfg.c_version); #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) if (signal(SIGSEGV, signal_handle_sigsegv) == SIG_ERR) { diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h index 4e1ae79..49cb7bf 100644 --- a/include/erofs/blobchunk.h +++ b/include/erofs/blobchunk.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * erofs-utils/lib/blobchunk.h * diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h index ca8053e..78fab44 100644 --- a/include/erofs/block_list.h +++ b/include/erofs/block_list.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu <huyue2@yulong.com> diff --git a/include/erofs/cache.h b/include/erofs/cache.h index 72b849d..de12399 100644 --- a/include/erofs/cache.h +++ b/include/erofs/cache.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ @@ -37,7 +37,7 @@ struct erofs_buffer_head { struct erofs_buffer_block *block; erofs_off_t off; - struct erofs_bhops *op; + const struct erofs_bhops *op; void *fsprivate; }; @@ -73,9 +73,9 @@ static inline const int get_alignsize(int type, int *type_ret) return -EINVAL; } -extern struct erofs_bhops erofs_drop_directly_bhops; -extern struct erofs_bhops erofs_skip_write_bhops; -extern struct erofs_bhops erofs_buf_write_bhops; +extern const struct erofs_bhops erofs_drop_directly_bhops; +extern const struct erofs_bhops erofs_skip_write_bhops; +extern const struct erofs_bhops erofs_buf_write_bhops; static inline erofs_off_t erofs_btell(struct erofs_buffer_head *bh, bool end) { diff --git a/include/erofs/compress.h b/include/erofs/compress.h index fdbf5ff..24f6204 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -14,11 +14,10 @@ extern "C" #include "internal.h" -/* workaround for an upstream lz4 compression issue, which can crash us */ -/* #define EROFS_CONFIG_COMPR_MAX_SZ (1024 * 1024) */ -#define EROFS_CONFIG_COMPR_MAX_SZ (900 * 1024) +#define EROFS_CONFIG_COMPR_MAX_SZ (3000 * 1024) #define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024) +void z_erofs_drop_inline_pcluster(struct erofs_inode *inode); int erofs_write_compressed_file(struct erofs_inode *inode); int z_erofs_compress_init(struct erofs_buffer_head *bh); diff --git a/include/erofs/compress_hints.h b/include/erofs/compress_hints.h index 43f80e1..659c5b6 100644 --- a/include/erofs/compress_hints.h +++ b/include/erofs/compress_hints.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan <huangjianan@oppo.com> diff --git a/include/erofs/config.h b/include/erofs/config.h index 0a1b18b..0d0916c 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -43,7 +43,9 @@ struct erofs_configure { char c_timeinherit; char c_chunkbits; bool c_noinline_data; + bool c_ztailpacking; bool c_ignore_mtime; + bool c_showprogress; #ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd; @@ -91,6 +93,9 @@ static inline int erofs_selabel_open(const char *file_contexts) } #endif +void erofs_update_progressinfo(const char *fmt, ...); +char *erofs_trim_for_progressinfo(const char *str, int placeholder); + #ifdef __cplusplus } #endif diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h index e649c80..82bf7b8 100644 --- a/include/erofs/decompress.h +++ b/include/erofs/decompress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan <huangjianan@oppo.com> diff --git a/include/erofs/defs.h b/include/erofs/defs.h index 4db237f..e5aa23c 100644 --- a/include/erofs/defs.h +++ b/include/erofs/defs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ @@ -61,7 +61,6 @@ typedef int16_t s16; typedef int32_t s32; typedef int64_t s64; - #if __BYTE_ORDER == __LITTLE_ENDIAN /* * The host byte order is the same as network byte order, diff --git a/include/erofs/dir.h b/include/erofs/dir.h index 77656ca..74bffb5 100644 --- a/include/erofs/dir.h +++ b/include/erofs/dir.h @@ -39,6 +39,15 @@ typedef int (*erofs_readdir_cb)(struct erofs_dir_context *); * the callback context. |de_namelen| is the exact dirent name length. */ struct erofs_dir_context { + /* + * During execution of |erofs_iterate_dir|, the function needs to + * read the values inside |erofs_inode* dir|. So it is important + * that the callback function does not modify struct pointed by + * |dir|. It is OK to repoint |dir| to other objects. + * Unfortunately, it's not possible to enforce this restriction + * with const keyword, as |erofs_iterate_dir| needs to modify + * struct pointed by |dir|. + */ struct erofs_inode *dir; erofs_readdir_cb cb; erofs_nid_t pnid; /* optional */ @@ -52,6 +61,8 @@ struct erofs_dir_context { /* Iterate over inodes that are in directory */ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck); +/* Get a full pathname of the inode NID */ +int erofs_get_pathname(erofs_nid_t nid, char *buf, size_t size); #ifdef __cplusplus } diff --git a/include/erofs/err.h b/include/erofs/err.h index 18f152a..08b0bdb 100644 --- a/include/erofs/err.h +++ b/include/erofs/err.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h index 599f018..3f17032 100644 --- a/include/erofs/exclude.h +++ b/include/erofs/exclude.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Created by Li Guifu <bluce.lee@aliyun.com> */ diff --git a/include/erofs/inode.h b/include/erofs/inode.h index e23d65f..79b39b0 100644 --- a/include/erofs/inode.h +++ b/include/erofs/inode.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 56627e9..6a70f11 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -36,7 +36,7 @@ typedef unsigned short umode_t; /* no obvious reason to support explicit PAGE_SIZE != 4096 for now */ #if PAGE_SIZE != 4096 -#error incompatible PAGE_SIZE is already defined +#warning EROFS may be incompatible on your platform #endif #ifndef PAGE_MASK @@ -131,6 +131,7 @@ EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) #define EROFS_I_EA_INITED (1 << 0) @@ -174,6 +175,7 @@ struct erofs_inode { unsigned char inode_isize; /* inline tail-end packing size */ unsigned short idata_size; + bool compressed_idata; unsigned int xattr_isize; unsigned int extent_isize; @@ -184,6 +186,10 @@ struct erofs_inode { void *idata; + /* (ztailpacking) in order to recover uncompressed EOF data */ + void *eof_tailraw; + unsigned int eof_tailrawsize; + union { void *compressmeta; void *chunkindexes; @@ -192,6 +198,9 @@ struct erofs_inode { uint8_t z_algorithmtype[2]; uint8_t z_logical_clusterbits; uint8_t z_physical_clusterblks; + uint64_t z_tailextent_headlcn; + unsigned int z_idataoff; +#define z_idata_size idata_size }; }; #ifdef WITH_ANDROID @@ -295,6 +304,7 @@ struct erofs_map_blocks { * approach instead if possible since it's more metadata lightweight.) */ #define EROFS_GET_BLOCKS_FIEMAP 0x0002 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0008 enum { Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, @@ -320,6 +330,27 @@ int erofs_pread(struct erofs_inode *inode, char *buf, int erofs_map_blocks(struct erofs_inode *inode, struct erofs_map_blocks *map, int flags); int erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map); + +static inline int erofs_get_occupied_size(const struct erofs_inode *inode, + erofs_off_t *size) +{ + *size = 0; + switch (inode->datalayout) { + case EROFS_INODE_FLAT_INLINE: + case EROFS_INODE_FLAT_PLAIN: + case EROFS_INODE_CHUNK_BASED: + *size = inode->i_size; + break; + case EROFS_INODE_FLAT_COMPRESSION_LEGACY: + case EROFS_INODE_FLAT_COMPRESSION: + *size = inode->u.i_blocks * EROFS_BLKSIZ; + break; + default: + return -ENOTSUP; + } + return 0; +} + /* zmap.c */ int z_erofs_fill_inode(struct erofs_inode *vi); int z_erofs_map_blocks_iter(struct erofs_inode *vi, diff --git a/include/erofs/io.h b/include/erofs/io.h index 6f51e06..0f58c70 100644 --- a/include/erofs/io.h +++ b/include/erofs/io.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/list.h b/include/erofs/list.h index fd5358d..3f5da1a 100644 --- a/include/erofs/list.h +++ b/include/erofs/list.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ @@ -110,7 +110,6 @@ static inline int list_empty(struct list_head *head) &pos->member != (head); \ pos = n, n = list_next_entry(n, member)) - #ifdef __cplusplus } #endif diff --git a/include/erofs/print.h b/include/erofs/print.h index 2213d1d..a896d75 100644 --- a/include/erofs/print.h +++ b/include/erofs/print.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -41,37 +41,39 @@ enum { #define PR_FMT_FUNC_LINE(fmt) pr_fmt(fmt), __func__, __LINE__ #endif +void erofs_msg(int dbglv, const char *fmt, ...); + #define erofs_dbg(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_DBG) { \ - fprintf(stdout, \ - "<D> " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_DBG, \ + "<D> " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ } \ } while (0) #define erofs_info(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_INFO) { \ - fprintf(stdout, \ - "<I> " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_INFO, \ + "<I> " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ fflush(stdout); \ } \ } while (0) #define erofs_warn(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_WARN) { \ - fprintf(stdout, \ - "<W> " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_WARN, \ + "<W> " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ fflush(stdout); \ } \ } while (0) #define erofs_err(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_ERR) { \ - fprintf(stderr, \ - "<E> " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_ERR, \ + "<E> " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ } \ } while (0) diff --git a/include/erofs/trace.h b/include/erofs/trace.h index 893e16c..398e331 100644 --- a/include/erofs/trace.h +++ b/include/erofs/trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2020 Gao Xiang <hsiangkao@aol.com> */ diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h index 8e68812..226e984 100644 --- a/include/erofs/xattr.h +++ b/include/erofs/xattr.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Originally contributed by an anonymous person, * heavily changed by Li Guifu <blucerlee@gmail.com> diff --git a/include/erofs_fs.h b/include/erofs_fs.h index 7956a62..08f9761 100644 --- a/include/erofs_fs.h +++ b/include/erofs_fs.h @@ -24,12 +24,14 @@ #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 #define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ - EROFS_FEATURE_INCOMPAT_DEVICE_TABLE) + EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ + EROFS_FEATURE_INCOMPAT_ZTAILPACKING) #define EROFS_SB_EXTSLOT_SIZE 16 @@ -291,13 +293,17 @@ struct z_erofs_lzma_cfgs { * (4B) + 2B + (4B) if compacted 2B is on. * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) + * bit 3 : tailpacking inline pcluster (0 - off; 1 - on) */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 struct z_erofs_map_header { - __le32 h_reserved1; + __le16 h_reserved1; + /* record the size of tailpacking data */ + __le16 h_idata_size; __le16 h_advise; /* * bit 0-3 : algorithm type of head 1 (logical cluster type 01); diff --git a/lib/Makefile.am b/lib/Makefile.am index 25a4a2b..3fad357 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0+ +# SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 noinst_LTLIBRARIES = liberofs.la noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ @@ -28,7 +28,7 @@ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c -liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS} liberofs_la_SOURCES += compressor_lz4.c diff --git a/lib/blobchunk.c b/lib/blobchunk.c index b605b0b..77b0c17 100644 --- a/lib/blobchunk.c +++ b/lib/blobchunk.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * erofs-utils/lib/blobchunk.c * @@ -113,7 +113,7 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, if (multidev) { idx.device_id = 1; - inode->u.chunkformat |= EROFS_CHUNK_FORMAT_INDEXES; + DBG_BUGON(!(inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES)); } else { base_blkaddr = remapped_base; } @@ -171,6 +171,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode) int fd, ret; inode->u.chunkformat |= inode->u.chunkbits - LOG_BLOCK_SIZE; + if (multidev) + inode->u.chunkformat |= EROFS_CHUNK_FORMAT_INDEXES; if (inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES) unit = sizeof(struct erofs_inode_chunk_index); @@ -219,6 +221,8 @@ int erofs_blob_remap(void) fflush(blobfile); length = ftell(blobfile); + if (length < 0) + return -errno; if (multidev) { struct erofs_deviceslot dis = { .blocks = erofs_blknr(length), diff --git a/lib/block_list.c b/lib/block_list.c index 87609a9..896fb01 100644 --- a/lib/block_list.c +++ b/lib/block_list.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu <huyue2@yulong.com> diff --git a/lib/cache.c b/lib/cache.c index 8016e38..c735363 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -26,7 +26,7 @@ static bool erofs_bh_flush_drop_directly(struct erofs_buffer_head *bh) return erofs_bh_flush_generic_end(bh); } -struct erofs_bhops erofs_drop_directly_bhops = { +const struct erofs_bhops erofs_drop_directly_bhops = { .flush = erofs_bh_flush_drop_directly, }; @@ -35,7 +35,7 @@ static bool erofs_bh_flush_skip_write(struct erofs_buffer_head *bh) return false; } -struct erofs_bhops erofs_skip_write_bhops = { +const struct erofs_bhops erofs_skip_write_bhops = { .flush = erofs_bh_flush_skip_write, }; @@ -58,7 +58,7 @@ static bool erofs_bh_flush_buf_write(struct erofs_buffer_head *bh) return erofs_bh_flush_generic_end(bh); } -struct erofs_bhops erofs_buf_write_bhops = { +const struct erofs_bhops erofs_buf_write_bhops = { .flush = erofs_bh_flush_buf_write, }; @@ -331,7 +331,6 @@ struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh, return ERR_PTR(ret); } return nbh; - } static erofs_blk_t __erofs_mapbh(struct erofs_buffer_block *bb) diff --git a/lib/compress.c b/lib/compress.c index 98be7a2..ee3b856 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -70,11 +70,15 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, di.di_clusterofs = cpu_to_le16(ctx->clusterofs); - /* whether the tail-end uncompressed block or not */ + /* whether the tail-end (un)compressed block or not */ if (!d1) { - /* TODO: tail-packing inline compressed data */ - DBG_BUGON(!raw); - type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN; + /* + * A lcluster cannot have three parts with the middle one which + * is well-compressed for !ztailpacking cases. + */ + DBG_BUGON(!raw && !cfg.c_ztailpacking); + type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN : + Z_EROFS_VLE_CLUSTER_TYPE_HEAD; advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT); di.di_advise = advise; @@ -97,7 +101,22 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, } else if (d0) { type = Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD; - di.di_u.delta[0] = cpu_to_le16(d0); + /* + * If the |Z_EROFS_VLE_DI_D0_CBLKCNT| bit is set, parser + * will interpret |delta[0]| as size of pcluster, rather + * than distance to last head cluster. Normally this + * isn't a problem, because uncompressed extent size are + * below Z_EROFS_VLE_DI_D0_CBLKCNT * BLOCK_SIZE = 8MB. + * But with large pcluster it's possible to go over this + * number, resulting in corrupted compressed indices. + * To solve this, we replace d0 with + * Z_EROFS_VLE_DI_D0_CBLKCNT-1. + */ + if (d0 >= Z_EROFS_VLE_DI_D0_CBLKCNT) + di.di_u.delta[0] = cpu_to_le16( + Z_EROFS_VLE_DI_D0_CBLKCNT - 1); + else + di.di_u.delta[0] = cpu_to_le16(d0); di.di_u.delta[1] = cpu_to_le16(d1); } else { type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN : @@ -162,6 +181,47 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode) return cfg.c_pclusterblks_def; } +static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data, + unsigned int len, bool raw) +{ + inode->z_advise |= Z_EROFS_ADVISE_INLINE_PCLUSTER; + inode->idata_size = len; + inode->compressed_idata = !raw; + + inode->idata = malloc(inode->idata_size); + if (!inode->idata) + return -ENOMEM; + erofs_dbg("Recording %u %scompressed inline data", + inode->idata_size, raw ? "un" : ""); + memcpy(inode->idata, data, inode->idata_size); + return len; +} + +static void tryrecompress_trailing(void *in, unsigned int *insize, + void *out, int *compressedsize) +{ + static char tmp[Z_EROFS_PCLUSTER_MAX_SIZE]; + unsigned int count; + int ret = *compressedsize; + + /* no need to recompress */ + if (!(ret & (EROFS_BLKSIZ - 1))) + return; + + count = *insize; + ret = erofs_compress_destsize(&compresshandle, + in, &count, (void *)tmp, + rounddown(ret, EROFS_BLKSIZ), false); + if (ret <= 0 || ret + (*insize - count) >= + roundup(*compressedsize, EROFS_BLKSIZ)) + return; + + /* replace the original compressed data if any gain */ + memcpy(out, tmp, ret); + *insize = count; + *compressedsize = ret; +} + static int vle_compress_one(struct erofs_inode *inode, struct z_erofs_vle_compress_ctx *ctx, bool final) @@ -174,41 +234,75 @@ static int vle_compress_one(struct erofs_inode *inode, char *const dst = dstbuf + EROFS_BLKSIZ; while (len) { - const unsigned int pclustersize = + unsigned int pclustersize = z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ; + bool may_inline = (cfg.c_ztailpacking && final); bool raw; if (len <= pclustersize) { - if (final) { - if (len <= EROFS_BLKSIZ) - goto nocompression; - } else { + if (!final) break; - } + if (!may_inline && len <= EROFS_BLKSIZ) + goto nocompression; } count = min(len, cfg.c_max_decompressed_extent_bytes); ret = erofs_compress_destsize(h, ctx->queue + ctx->head, - &count, dst, pclustersize); + &count, dst, pclustersize, + !(final && len == count)); if (ret <= 0) { if (ret != -EAGAIN) { erofs_err("failed to compress %s: %s", inode->i_srcpath, erofs_strerror(ret)); } + + if (may_inline && len < EROFS_BLKSIZ) + ret = z_erofs_fill_inline_data(inode, + ctx->queue + ctx->head, + len, true); + else nocompression: - ret = write_uncompressed_extent(ctx, &len, dst); + ret = write_uncompressed_extent(ctx, &len, dst); + if (ret < 0) return ret; count = ret; + + /* + * XXX: For now, we have to leave `ctx->compressedblks + * = 1' since there is no way to generate compressed + * indexes after the time that ztailpacking is decided. + */ ctx->compressedblks = 1; raw = true; + /* tailpcluster should be less than 1 block */ + } else if (may_inline && len == count && + ret < EROFS_BLKSIZ) { + if (ctx->clusterofs + len <= EROFS_BLKSIZ) { + inode->eof_tailraw = malloc(len); + if (!inode->eof_tailraw) + return -ENOMEM; + + memcpy(inode->eof_tailraw, + ctx->queue + ctx->head, len); + inode->eof_tailrawsize = len; + } + + ret = z_erofs_fill_inline_data(inode, dst, ret, false); + if (ret < 0) + return ret; + ctx->compressedblks = 1; + raw = false; } else { - const unsigned int tailused = ret & (EROFS_BLKSIZ - 1); - const unsigned int padding = - erofs_sb_has_lz4_0padding() && tailused ? - EROFS_BLKSIZ - tailused : 0; + unsigned int tailused, padding; + if (may_inline && len == count) + tryrecompress_trailing(ctx->queue + ctx->head, + &count, dst, &ret); + + tailused = ret & (EROFS_BLKSIZ - 1); + padding = 0; ctx->compressedblks = DIV_ROUND_UP(ret, EROFS_BLKSIZ); DBG_BUGON(ctx->compressedblks * EROFS_BLKSIZ >= count); @@ -216,6 +310,8 @@ nocompression: if (!erofs_sb_has_lz4_0padding()) memset(dst + ret, 0, roundup(ret, EROFS_BLKSIZ) - ret); + else if (tailused) + padding = EROFS_BLKSIZ - tailused; /* write compressed data */ erofs_dbg("Writing %u compressed data to %u of %u blocks", @@ -359,7 +455,8 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode, inode->xattr_isize) + sizeof(struct z_erofs_map_header); const unsigned int totalidx = (legacymetasize - - Z_EROFS_LEGACY_MAP_HEADER_SIZE) / 8; + Z_EROFS_LEGACY_MAP_HEADER_SIZE) / + sizeof(struct z_erofs_vle_decompressed_index); const unsigned int logical_clusterbits = inode->z_logical_clusterbits; u8 *out, *in; struct z_erofs_compressindex_vec cv[16]; @@ -449,6 +546,7 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, { struct z_erofs_map_header h = { .h_advise = cpu_to_le16(inode->z_advise), + .h_idata_size = cpu_to_le16(inode->idata_size), .h_algorithmtype = inode->z_algorithmtype[1] << 4 | inode->z_algorithmtype[0], /* lclustersize */ @@ -460,10 +558,56 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, memcpy(compressmeta, &h, sizeof(struct z_erofs_map_header)); } +void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) +{ + const unsigned int type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN; + struct z_erofs_map_header *h = inode->compressmeta; + + h->h_advise = cpu_to_le16(le16_to_cpu(h->h_advise) & + ~Z_EROFS_ADVISE_INLINE_PCLUSTER); + if (!inode->eof_tailraw) + return; + DBG_BUGON(inode->compressed_idata != true); + + /* patch the EOF lcluster to uncompressed type first */ + if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { + struct z_erofs_vle_decompressed_index *di = + (inode->compressmeta + inode->extent_isize) - + sizeof(struct z_erofs_vle_decompressed_index); + __le16 advise = + cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT); + + di->di_advise = advise; + } else if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION) { + /* handle the last compacted 4B pack */ + unsigned int eofs, base, pos, v, lo; + u8 *out; + + eofs = inode->extent_isize - + (4 << (DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ) & 1)); + base = round_down(eofs, 8); + pos = 16 /* encodebits */ * ((eofs - base) / 4); + out = inode->compressmeta + base; + lo = get_unaligned_le32(out + pos / 8) & (EROFS_BLKSIZ - 1); + v = (type << LOG_BLOCK_SIZE) | lo; + out[pos / 8] = v & 0xff; + out[pos / 8 + 1] = v >> 8; + } else { + DBG_BUGON(1); + return; + } + free(inode->idata); + /* replace idata with prepared uncompressed data */ + inode->idata = inode->eof_tailraw; + inode->idata_size = inode->eof_tailrawsize; + inode->compressed_idata = false; + inode->eof_tailraw = NULL; +} + int erofs_write_compressed_file(struct erofs_inode *inode) { struct erofs_buffer_head *bh; - struct z_erofs_vle_compress_ctx ctx; + static struct z_erofs_vle_compress_ctx ctx; erofs_off_t remaining; erofs_blk_t blkaddr, compressed_blocks; unsigned int legacymetasize; @@ -476,7 +620,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) fd = open(inode->i_srcpath, O_RDONLY | O_BINARY); if (fd < 0) { ret = -errno; - goto err_free; + goto err_free_meta; } /* allocate main data buffer */ @@ -504,8 +648,6 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->z_algorithmtype[1] = algorithmtype[1]; inode->z_logical_clusterbits = LOG_BLOCK_SIZE; - z_erofs_write_mapheader(inode, compressmeta); - blkaddr = erofs_mapbh(bh->block); /* start_blkaddr */ ctx.blkaddr = blkaddr; ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE; @@ -525,44 +667,46 @@ int erofs_write_compressed_file(struct erofs_inode *inode) remaining -= readcount; ctx.tail += readcount; - /* do one compress round */ - ret = vle_compress_one(inode, &ctx, false); + ret = vle_compress_one(inode, &ctx, !remaining); if (ret) - goto err_bdrop; + goto err_free_idata; } - - /* do the final round */ - ret = vle_compress_one(inode, &ctx, true); - if (ret) - goto err_bdrop; + DBG_BUGON(ctx.head != ctx.tail); /* fall back to no compression mode */ compressed_blocks = ctx.blkaddr - blkaddr; - if (compressed_blocks >= BLK_ROUND_UP(inode->i_size)) { - ret = -ENOSPC; - goto err_bdrop; - } + DBG_BUGON(compressed_blocks < !!inode->idata_size); + compressed_blocks -= !!inode->idata_size; vle_write_indexes_final(&ctx); + legacymetasize = ctx.metacur - compressmeta; + /* estimate if data compression saves space or not */ + if (compressed_blocks * EROFS_BLKSIZ + inode->idata_size + + legacymetasize >= inode->i_size) { + ret = -ENOSPC; + goto err_free_idata; + } + z_erofs_write_mapheader(inode, compressmeta); close(fd); - DBG_BUGON(!compressed_blocks); - ret = erofs_bh_balloon(bh, blknr_to_addr(compressed_blocks)); - DBG_BUGON(ret != EROFS_BLKSIZ); + if (compressed_blocks) { + ret = erofs_bh_balloon(bh, blknr_to_addr(compressed_blocks)); + DBG_BUGON(ret != EROFS_BLKSIZ); + } else { + DBG_BUGON(!inode->idata_size); + } erofs_info("compressed %s (%llu bytes) into %u blocks", inode->i_srcpath, (unsigned long long)inode->i_size, compressed_blocks); - /* - * TODO: need to move erofs_bdrop to erofs_write_tail_end - * when both mkfs & kernel support compression inline. - */ - erofs_bdrop(bh, false); - inode->idata_size = 0; + if (inode->idata_size) + inode->bh_data = bh; + else + erofs_bdrop(bh, false); + inode->u.i_blocks = compressed_blocks; - legacymetasize = ctx.metacur - compressmeta; if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { inode->extent_isize = legacymetasize; } else { @@ -575,11 +719,16 @@ int erofs_write_compressed_file(struct erofs_inode *inode) erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks); return 0; +err_free_idata: + if (inode->idata) { + free(inode->idata); + inode->idata = NULL; + } err_bdrop: erofs_bdrop(bh, true); /* revoke buffer */ err_close: close(fd); -err_free: +err_free_meta: free(compressmeta); return ret; } @@ -691,7 +840,6 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) return -EINVAL; } erofs_sb_set_big_pcluster(); - erofs_warn("EXPERIMENTAL big pcluster feature in use. Use at your own risk!"); } if (ret != Z_EROFS_COMPRESSION_LZ4) diff --git a/lib/compress_hints.c b/lib/compress_hints.c index 81a8ac9..92964eb 100644 --- a/lib/compress_hints.c +++ b/lib/compress_hints.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan <huangjianan@oppo.com> @@ -88,6 +88,7 @@ int erofs_load_compress_hints(void) char buf[PATH_MAX + 100]; FILE *f; unsigned int line, max_pclustersize = 0; + int ret = 0; if (!cfg.c_compress_hints_file) return 0; @@ -105,7 +106,8 @@ int erofs_load_compress_hints(void) if (!pattern || *pattern == '\0') { erofs_err("cannot find a match pattern at line %u", line); - return -EINVAL; + ret = -EINVAL; + goto out; } if (pclustersize % EROFS_BLKSIZ) { erofs_warn("invalid physical clustersize %u, " @@ -119,10 +121,12 @@ int erofs_load_compress_hints(void) if (pclustersize > max_pclustersize) max_pclustersize = pclustersize; } - fclose(f); + if (cfg.c_pclusterblks_max * EROFS_BLKSIZ < max_pclustersize) { cfg.c_pclusterblks_max = max_pclustersize / EROFS_BLKSIZ; erofs_warn("update max pclusterblks to %u", cfg.c_pclusterblks_max); } - return 0; +out: + fclose(f); + return ret; } diff --git a/lib/compressor.c b/lib/compressor.c index ad12cdf..a46bc39 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -10,7 +10,7 @@ #define EROFS_CONFIG_COMPR_DEF_BOUNDARY (128) -static struct erofs_compressor *compressors[] = { +static const struct erofs_compressor *compressors[] = { #if LZ4_ENABLED #if LZ4HC_ENABLED &erofs_compressor_lz4hc, @@ -22,25 +22,31 @@ static struct erofs_compressor *compressors[] = { #endif }; -int erofs_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, - void *dst, unsigned int dstsize) +int erofs_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, + void *dst, unsigned int dstsize, bool inblocks) { - unsigned int uncompressed_size; + unsigned int uncompressed_capacity, compressed_size; int ret; DBG_BUGON(!c->alg); if (!c->alg->compress_destsize) return -ENOTSUP; + uncompressed_capacity = *srcsize; ret = c->alg->compress_destsize(c, src, srcsize, dst, dstsize); if (ret < 0) return ret; + /* XXX: ret >= EROFS_BLKSIZ is a temporary hack for ztailpacking */ + if (inblocks || ret >= EROFS_BLKSIZ || + uncompressed_capacity != *srcsize) + compressed_size = roundup(ret, EROFS_BLKSIZ); + else + compressed_size = ret; + DBG_BUGON(c->compress_threshold < 100); /* check if there is enough gains to compress */ - uncompressed_size = *srcsize; - if (roundup(ret, EROFS_BLKSIZ) >= uncompressed_size * - c->compress_threshold / 100) + if (*srcsize <= compressed_size * c->compress_threshold / 100) return -EAGAIN; return ret; } @@ -70,8 +76,8 @@ int erofs_compressor_init(struct erofs_compress *c, char *alg_name) c->compress_threshold = 100; /* optimize for 4k size page */ - c->destsize_alignsize = PAGE_SIZE; - c->destsize_redzone_begin = PAGE_SIZE - 16; + c->destsize_alignsize = EROFS_BLKSIZ; + c->destsize_redzone_begin = EROFS_BLKSIZ - 16; c->destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY; if (!alg_name) { diff --git a/lib/compressor.h b/lib/compressor.h index aa85ae0..cf063f1 100644 --- a/lib/compressor.h +++ b/lib/compressor.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -21,13 +21,13 @@ struct erofs_compressor { int (*exit)(struct erofs_compress *c); int (*setlevel)(struct erofs_compress *c, int compression_level); - int (*compress_destsize)(struct erofs_compress *c, - void *src, unsigned int *srcsize, + int (*compress_destsize)(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize); }; struct erofs_compress { - struct erofs_compressor *alg; + const struct erofs_compressor *alg; unsigned int compress_threshold; unsigned int compression_level; @@ -41,13 +41,13 @@ struct erofs_compress { }; /* list of compression algorithms */ -extern struct erofs_compressor erofs_compressor_lz4; -extern struct erofs_compressor erofs_compressor_lz4hc; -extern struct erofs_compressor erofs_compressor_lzma; +extern const struct erofs_compressor erofs_compressor_lz4; +extern const struct erofs_compressor erofs_compressor_lz4hc; +extern const struct erofs_compressor erofs_compressor_lzma; -int erofs_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, - void *dst, unsigned int dstsize); +int erofs_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, + void *dst, unsigned int dstsize, bool inblocks); int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level); int erofs_compressor_init(struct erofs_compress *c, char *alg_name); diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c index 40a05ef..4886d6a 100644 --- a/lib/compressor_liblzma.c +++ b/lib/compressor_liblzma.c @@ -1,7 +1,5 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* - * erofs-utils/lib/compressor_liblzma.c - * * Copyright (C) 2021 Gao Xiang <xiang@kernel.org> */ #include <stdlib.h> @@ -18,8 +16,8 @@ struct erofs_liblzma_context { lzma_stream strm; }; -static int erofs_liblzma_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int erofs_liblzma_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { struct erofs_liblzma_context *ctx = c->private_data; @@ -96,7 +94,7 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c) return 0; } -struct erofs_compressor erofs_compressor_lzma = { +const struct erofs_compressor erofs_compressor_lzma = { .name = "lzma", .default_level = LZMA_PRESET_DEFAULT, .best_level = LZMA_PRESET_EXTREME, diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c index f6832be..b6f6e7e 100644 --- a/lib/compressor_lz4.c +++ b/lib/compressor_lz4.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -12,8 +12,8 @@ #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif -static int lz4_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int lz4_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { int srcSize = (int)*srcsize; @@ -37,7 +37,7 @@ static int compressor_lz4_init(struct erofs_compress *c) return 0; } -struct erofs_compressor erofs_compressor_lz4 = { +const struct erofs_compressor erofs_compressor_lz4 = { .name = "lz4", .default_level = 0, .best_level = 0, diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c index fd801ab..eec1c84 100644 --- a/lib/compressor_lz4hc.c +++ b/lib/compressor_lz4hc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -13,8 +13,8 @@ #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif -static int lz4hc_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int lz4hc_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { int srcSize = (int)*srcsize; @@ -59,7 +59,7 @@ static int compressor_lz4hc_setlevel(struct erofs_compress *c, return 0; } -struct erofs_compressor erofs_compressor_lz4hc = { +const struct erofs_compressor erofs_compressor_lz4hc = { .name = "lz4hc", .default_level = LZ4HC_CLEVEL_DEFAULT, .best_level = LZ4HC_CLEVEL_MAX, diff --git a/lib/config.c b/lib/config.c index cc15e57..d478b07 100644 --- a/lib/config.c +++ b/lib/config.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -6,9 +6,13 @@ */ #include <string.h> #include <stdlib.h> +#include <stdarg.h> #include "erofs/print.h" #include "erofs/internal.h" #include "liberofs_private.h" +#ifdef HAVE_SYS_IOCTL_H +#include <sys/ioctl.h> +#endif struct erofs_configure cfg; struct erofs_sb_info sbi; @@ -36,7 +40,7 @@ void erofs_show_config(void) { const struct erofs_configure *c = &cfg; - if (c->c_dbg_lvl < EROFS_WARN) + if (c->c_dbg_lvl < EROFS_INFO) return; erofs_dump("\tc_version: [%8s]\n", c->c_version); erofs_dump("\tc_dbg_lvl: [%8d]\n", c->c_dbg_lvl); @@ -91,3 +95,66 @@ int erofs_selabel_open(const char *file_contexts) return 0; } #endif + +static bool __erofs_is_progressmsg; + +char *erofs_trim_for_progressinfo(const char *str, int placeholder) +{ + int col, len; + +#ifdef GWINSZ_IN_SYS_IOCTL + struct winsize winsize; + if(ioctl(1, TIOCGWINSZ, &winsize) >= 0 && + winsize.ws_col > 0) + col = winsize.ws_col; + else +#endif + col = 80; + + if (col <= placeholder) + return strdup(""); + + len = strlen(str); + /* omit over long prefixes */ + if (len > col - placeholder) { + char *s = strdup(str + len - (col - placeholder)); + + if (col > placeholder + 2) { + s[0] = '['; + s[1] = ']'; + } + return s; + } + return strdup(str); +} + +void erofs_msg(int dbglv, const char *fmt, ...) +{ + va_list ap; + FILE *f = dbglv >= EROFS_ERR ? stderr : stdout; + + if (__erofs_is_progressmsg) { + fputc('\n', f); + __erofs_is_progressmsg = false; + } + va_start(ap, fmt); + vfprintf(f, fmt, ap); + va_end(ap); +} + +void erofs_update_progressinfo(const char *fmt, ...) +{ + char msg[8192]; + va_list ap; + + if (cfg.c_dbg_lvl >= EROFS_INFO || !cfg.c_showprogress) + return; + + va_start(ap, fmt); + vsprintf(msg, fmt, ap); + va_end(ap); + + printf("\r\033[K%s", msg); + __erofs_is_progressmsg = true; + fflush(stdout); +} @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2020 Gao Xiang <hsiangkao@aol.com> * Compression support by Huang Jianan <huangjianan@oppo.com> @@ -22,7 +22,7 @@ static int erofs_map_blocks_flatmode(struct erofs_inode *inode, trace_erofs_map_blocks_flatmode_enter(inode, map, flags); - nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); + nblocks = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); lastblk = nblocks - tailendpacking; /* there is no hole in flatmode */ @@ -37,8 +37,8 @@ static int erofs_map_blocks_flatmode(struct erofs_inode *inode, vi->xattr_isize + erofs_blkoff(map->m_la); map->m_plen = inode->i_size - offset; - /* inline data should be located in one meta block */ - if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) { + /* inline data should be located in the same meta block */ + if (erofs_blkoff(map->m_pa) + map->m_plen > EROFS_BLKSIZ) { erofs_err("inline data cross block boundary @ nid %" PRIu64, vi->nid); DBG_BUGON(1); diff --git a/lib/decompress.c b/lib/decompress.c index f313e41..1661f91 100644 --- a/lib/decompress.c +++ b/lib/decompress.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan <huangjianan@oppo.com> @@ -110,6 +110,9 @@ static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq) rq->decodedlength); if (ret != (int)rq->decodedlength) { + erofs_err("failed to %s decompress %d in[%u, %u] out[%u]", + rq->partial_decoding ? "partial" : "full", + ret, rq->inputsize, inputmargin, rq->decodedlength); ret = -EIO; goto out; } @@ -129,7 +132,7 @@ out: int z_erofs_decompress(struct z_erofs_decompress_req *rq) { if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) { - if (rq->inputsize != EROFS_BLKSIZ) + if (rq->inputsize > EROFS_BLKSIZ) return -EFSCORRUPTED; DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ); @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 -#include "erofs/print.h" -#include "erofs/dir.h" #include <stdlib.h> #include <sys/stat.h> +#include "erofs/print.h" +#include "erofs/dir.h" static int traverse_dirents(struct erofs_dir_context *ctx, void *dentry_blk, unsigned int lblk, @@ -127,7 +127,7 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) erofs_off_t pos; char buf[EROFS_BLKSIZ]; - if ((dir->i_mode & S_IFMT) != S_IFDIR) + if (!S_ISDIR(dir->i_mode)) return -ENOTDIR; ctx->flags &= ~EROFS_READDIR_ALL_SPECIAL_FOUND; @@ -148,7 +148,7 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) nameoff = le16_to_cpu(de->nameoff); if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { + nameoff >= EROFS_BLKSIZ) { erofs_err("invalid de[0].nameoff %u @ nid %llu, lblk %u", nameoff, dir->nid | 0ULL, lblk); return -EFSCORRUPTED; @@ -167,3 +167,104 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) } return err; } + +#define EROFS_PATHNAME_FOUND 1 + +struct erofs_get_pathname_context { + struct erofs_dir_context ctx; + erofs_nid_t target_nid; + char *buf; + size_t size; + size_t pos; +}; + +static int erofs_get_pathname_iter(struct erofs_dir_context *ctx) +{ + int ret; + struct erofs_get_pathname_context *pathctx = (void *)ctx; + const char *dname = ctx->dname; + size_t len = ctx->de_namelen; + size_t pos = pathctx->pos; + + if (ctx->dot_dotdot) + return 0; + + if (ctx->de_nid == pathctx->target_nid) { + if (pos + len + 2 > pathctx->size) { + erofs_err("get_pathname buffer not large enough: len %zd, size %zd", + pos + len + 2, pathctx->size); + return -ERANGE; + } + + pathctx->buf[pos++] = '/'; + strncpy(pathctx->buf + pos, dname, len); + pathctx->buf[pos + len] = '\0'; + return EROFS_PATHNAME_FOUND; + } + + if (ctx->de_ftype == EROFS_FT_DIR || ctx->de_ftype == EROFS_FT_UNKNOWN) { + struct erofs_inode dir = { .nid = ctx->de_nid }; + + ret = erofs_read_inode_from_disk(&dir); + if (ret) { + erofs_err("read inode failed @ nid %llu", dir.nid | 0ULL); + return ret; + } + + if (S_ISDIR(dir.i_mode)) { + ctx->dir = &dir; + pathctx->pos = pos + len + 1; + ret = erofs_iterate_dir(ctx, false); + pathctx->pos = pos; + if (ret == EROFS_PATHNAME_FOUND) { + pathctx->buf[pos++] = '/'; + strncpy(pathctx->buf + pos, dname, len); + } + return ret; + } else if (ctx->de_ftype == EROFS_FT_DIR) { + erofs_err("i_mode and file_type are inconsistent @ nid %llu", + dir.nid | 0ULL); + } + } + return 0; +} + +int erofs_get_pathname(erofs_nid_t nid, char *buf, size_t size) +{ + int ret; + struct erofs_inode root = { .nid = sbi.root_nid }; + struct erofs_get_pathname_context pathctx = { + .ctx.flags = 0, + .ctx.dir = &root, + .ctx.cb = erofs_get_pathname_iter, + .target_nid = nid, + .buf = buf, + .size = size, + .pos = 0, + }; + + if (nid == root.nid) { + if (size < 2) { + erofs_err("get_pathname buffer not large enough: len 2, size %zd", + size); + return -ERANGE; + } + + buf[0] = '/'; + buf[1] = '\0'; + return 0; + } + + ret = erofs_read_inode_from_disk(&root); + if (ret) { + erofs_err("read inode failed @ nid %llu", root.nid | 0ULL); + return ret; + } + + ret = erofs_iterate_dir(&pathctx.ctx, false); + if (ret == EROFS_PATHNAME_FOUND) + return 0; + if (!ret) + return -ENOENT; + return ret; +} diff --git a/lib/exclude.c b/lib/exclude.c index 2f980a3..e3c4ed5 100644 --- a/lib/exclude.c +++ b/lib/exclude.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu <bluce.lee@aliyun.com> */ diff --git a/lib/inode.c b/lib/inode.c index 77ea8bf..f192510 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ @@ -96,6 +96,8 @@ unsigned int erofs_iput(struct erofs_inode *inode) list_for_each_entry_safe(d, t, &inode->i_subdirs, d_child) free(d); + if (inode->eof_tailraw) + free(inode->eof_tailraw); list_del(&inode->i_hash); free(inode); return 0; @@ -593,28 +595,31 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode) inodesize = Z_EROFS_VLE_EXTENT_ALIGN(inodesize) + inode->extent_isize; - if (is_inode_layout_compression(inode)) - goto noinline; + /* TODO: tailpacking inline of chunk-based format isn't finalized */ if (inode->datalayout == EROFS_INODE_CHUNK_BASED) goto noinline; - if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) { - inode->datalayout = EROFS_INODE_FLAT_PLAIN; - goto noinline; + if (!is_inode_layout_compression(inode)) { + if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) { + inode->datalayout = EROFS_INODE_FLAT_PLAIN; + goto noinline; + } + /* + * If the file sizes of uncompressed files are block-aligned, + * should use the EROFS_INODE_FLAT_PLAIN data layout. + */ + if (!inode->idata_size) + inode->datalayout = EROFS_INODE_FLAT_PLAIN; } - /* - * if the file size is block-aligned for uncompressed files, - * should use EROFS_INODE_FLAT_PLAIN data mapping mode. - */ - if (!inode->idata_size) - inode->datalayout = EROFS_INODE_FLAT_PLAIN; - bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size); if (bh == ERR_PTR(-ENOSPC)) { int ret; - inode->datalayout = EROFS_INODE_FLAT_PLAIN; + if (is_inode_layout_compression(inode)) + z_erofs_drop_inline_pcluster(inode); + else + inode->datalayout = EROFS_INODE_FLAT_PLAIN; noinline: /* expend an extra block for tail-end data */ ret = erofs_prepare_tail_block(inode); @@ -627,7 +632,17 @@ noinline: } else if (IS_ERR(bh)) { return PTR_ERR(bh); } else if (inode->idata_size) { - inode->datalayout = EROFS_INODE_FLAT_INLINE; + if (is_inode_layout_compression(inode)) { + DBG_BUGON(!cfg.c_ztailpacking); + erofs_dbg("Inline %scompressed data (%u bytes) to %s", + inode->compressed_idata ? "" : "un", + inode->idata_size, inode->i_srcpath); + erofs_sb_set_ztailpacking(); + } else { + inode->datalayout = EROFS_INODE_FLAT_INLINE; + erofs_dbg("Inline tail-end data (%u bytes) to %s", + inode->idata_size, inode->i_srcpath); + } /* allocate inline buffer */ ibh = erofs_battach(bh, META, inode->idata_size); @@ -685,15 +700,26 @@ static int erofs_write_tail_end(struct erofs_inode *inode) erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR); } else { int ret; - erofs_off_t pos; + erofs_off_t pos, zero_pos; erofs_mapbh(bh->block); pos = erofs_btell(bh, true) - EROFS_BLKSIZ; + + /* 0'ed data should be padded at head for 0padding conversion */ + if (erofs_sb_has_lz4_0padding() && inode->compressed_idata) { + zero_pos = pos; + pos += EROFS_BLKSIZ - inode->idata_size; + } else { + /* pad 0'ed data for the other cases */ + zero_pos = pos + inode->idata_size; + } ret = dev_write(inode->idata, pos, inode->idata_size); if (ret) return ret; + + DBG_BUGON(inode->idata_size > EROFS_BLKSIZ); if (inode->idata_size < EROFS_BLKSIZ) { - ret = dev_fillzero(pos + inode->idata_size, + ret = dev_fillzero(zero_pos, EROFS_BLKSIZ - inode->idata_size, false); if (ret) @@ -1057,7 +1083,7 @@ static struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir) erofs_fixup_meta_blkaddr(dir); list_for_each_entry(d, &dir->i_subdirs, d_child) { - char buf[PATH_MAX]; + char buf[PATH_MAX], *trimmed; unsigned char ftype; if (is_dot_dotdot(d->name)) { @@ -1072,6 +1098,10 @@ static struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir) goto fail; } + trimmed = erofs_trim_for_progressinfo(erofs_fspath(buf), + sizeof("Processing ...") - 1); + erofs_update_progressinfo("Processing %s ...", trimmed); + free(trimmed); d->inode = erofs_mkfs_build_tree_from_path(dir, buf); if (IS_ERR(d->inode)) { ret = PTR_ERR(d->inode); @@ -1086,7 +1116,7 @@ fail: d->type = ftype; erofs_d_invalidate(d); - erofs_info("add file %s/%s (nid %llu, type %d)", + erofs_info("add file %s/%s (nid %llu, type %u)", dir->i_srcpath, d->name, (unsigned long long)d->nid, d->type); } @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ @@ -370,7 +370,7 @@ ssize_t erofs_copy_file_range(int fd_in, erofs_off_t *off_in, ssize_t ret; ret = copy_file_range(fd_in, &off64_in, fd_out, &off64_out, - length, 0); + length, 0); if (ret >= 0) goto out; if (errno != ENOSYS) { diff --git a/lib/liberofs_private.h b/lib/liberofs_private.h index c2312e8..0eeca3c 100644 --- a/lib/liberofs_private.h +++ b/lib/liberofs_private.h @@ -11,3 +11,15 @@ #include <private/canned_fs_config.h> #include <private/fs_config.h> #endif + +#ifndef HAVE_MEMRCHR +static inline void *memrchr(const void *s, int c, size_t n) +{ + const unsigned char *p = (const unsigned char *)s; + + for (p += n; n > 0; n--) + if (*--p == c) + return (void*)p; + return NULL; +} +#endif diff --git a/lib/namei.c b/lib/namei.c index 2c8891a..7b69a59 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu <blucerlee@gmail.com> */ @@ -137,14 +137,13 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->u.chunkbits = LOG_BLOCK_SIZE + (vi->u.chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); } else if (erofs_inode_is_data_compressed(vi->datalayout)) - z_erofs_fill_inode(vi); + return z_erofs_fill_inode(vi); return 0; bogusimode: erofs_err("bogus i_mode (%o) @ nid %llu", vi->i_mode, vi->nid | 0ULL); return -EFSCORRUPTED; } - struct erofs_dirent *find_target_dirent(erofs_nid_t pnid, void *dentry_blk, const char *name, unsigned int len, @@ -213,7 +212,7 @@ int erofs_namei(struct nameidata *nd, nameoff = le16_to_cpu(de->nameoff); if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { + nameoff >= EROFS_BLKSIZ) { erofs_err("invalid de[0].nameoff %u @ nid %llu", nameoff, nid | 0ULL); return -EFSCORRUPTED; diff --git a/lib/super.c b/lib/super.c index 3ccc551..f486eb7 100644 --- a/lib/super.c +++ b/lib/super.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu <blucerlee@gmail.com> */ @@ -15,7 +15,7 @@ static bool check_layout_compatibility(struct erofs_sb_info *sbi, sbi->feature_incompat = feature; /* check if current kernel meets all mandatory requirements */ - if (feature & (~EROFS_ALL_FEATURE_INCOMPAT)) { + if (feature & ~EROFS_ALL_FEATURE_INCOMPAT) { erofs_err("unidentified incompatible feature %x, please upgrade kernel version", feature & ~EROFS_ALL_FEATURE_INCOMPAT); return false; @@ -87,7 +87,7 @@ int erofs_read_superblock(void) blkszbits = dsb->blkszbits; /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ if (blkszbits != LOG_BLOCK_SIZE) { - erofs_err("blksize %u isn't supported on this platform", + erofs_err("blksize %d isn't supported on this platform", 1 << blkszbits); return ret; } diff --git a/lib/xattr.c b/lib/xattr.c index 00fb963..71ffe3e 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Originally contributed by an anonymous person, * heavily changed by Li Guifu <blucerlee@gmail.com> @@ -563,13 +563,31 @@ static struct erofs_bhops erofs_write_shared_xattrs_bhops = { .flush = erofs_bh_flush_write_shared_xattrs, }; +static int comp_xattr_item(const void *a, const void *b) +{ + const struct xattr_item *ia, *ib; + unsigned int la, lb; + int ret; + + ia = (*((const struct inode_xattr_node **)a))->item; + ib = (*((const struct inode_xattr_node **)b))->item; + la = ia->len[0] + ia->len[1]; + lb = ib->len[0] + ib->len[1]; + + ret = strncmp(ia->kvbuf, ib->kvbuf, min(la, lb)); + if (ret != 0) + return ret; + + return la > lb; +} + int erofs_build_shared_xattrs_from_path(const char *path) { int ret; struct erofs_buffer_head *bh; - struct inode_xattr_node *node, *n; + struct inode_xattr_node *node, *n, **sorted_n; char *buf; - unsigned int p; + unsigned int p, i; erofs_off_t off; /* check if xattr or shared xattr is disabled */ @@ -607,16 +625,26 @@ int erofs_build_shared_xattrs_from_path(const char *path) off %= EROFS_BLKSIZ; p = 0; + sorted_n = malloc(shared_xattrs_count * sizeof(n)); + if (!sorted_n) + return -ENOMEM; + i = 0; list_for_each_entry_safe(node, n, &shared_xattrs_list, list) { - struct xattr_item *const item = node->item; + list_del(&node->list); + sorted_n[i++] = node; + } + DBG_BUGON(i != shared_xattrs_count); + qsort(sorted_n, shared_xattrs_count, sizeof(n), comp_xattr_item); + + for (i = 0; i < shared_xattrs_count; i++) { + struct inode_xattr_node *const tnode = sorted_n[i]; + struct xattr_item *const item = tnode->item; const struct erofs_xattr_entry entry = { .e_name_index = item->prefix, .e_name_len = item->len[0], .e_value_size = cpu_to_le16(item->len[1]) }; - list_del(&node->list); - item->shared_xattr_id = (off + p) / sizeof(struct erofs_xattr_entry); @@ -624,8 +652,10 @@ int erofs_build_shared_xattrs_from_path(const char *path) p += sizeof(struct erofs_xattr_entry); memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]); p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]); - free(node); + free(tnode); } + + free(sorted_n); bh->fsprivate = buf; bh->op = &erofs_write_shared_xattrs_bhops; out: @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * (a large amount of code was adapted from Linux kernel. ) * @@ -10,9 +10,14 @@ #include "erofs/io.h" #include "erofs/print.h" +static int z_erofs_do_map_blocks(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags); + int z_erofs_fill_inode(struct erofs_inode *vi) { if (!erofs_sb_has_big_pcluster() && + !erofs_sb_has_ztailpacking() && vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { vi->z_advise = 0; vi->z_algorithmtype[0] = 0; @@ -35,6 +40,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi) return 0; DBG_BUGON(!erofs_sb_has_big_pcluster() && + !erofs_sb_has_ztailpacking() && vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); pos = round_up(iloc(vi->nid) + vi->inode_isize + vi->xattr_isize, 8); @@ -61,6 +67,22 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi) vi->nid * 1ULL); return -EFSCORRUPTED; } + + if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { + struct erofs_map_blocks map = { .index = UINT_MAX }; + + vi->idata_size = le16_to_cpu(h->h_idata_size); + ret = z_erofs_do_map_blocks(vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (!map.m_plen || + erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) { + erofs_err("invalid tail-packing pclustersize %llu", + map.m_plen | 0ULL); + return -EFSCORRUPTED; + } + if (ret < 0) + return ret; + } vi->flags |= EROFS_I_Z_INITED; return 0; } @@ -76,6 +98,7 @@ struct z_erofs_maprecorder { u16 clusterofs; u16 delta[2]; erofs_blk_t pblk, compressedlcs; + erofs_off_t nextpackoff; }; static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, @@ -114,6 +137,7 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, if (err) return err; + m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); m->lcn = lcn; di = m->kaddr + erofs_blkoff(pos); @@ -186,12 +210,12 @@ static int get_compacted_la_distance(unsigned int lclusterbits, static int unpack_compacted_index(struct z_erofs_maprecorder *m, unsigned int amortizedshift, - unsigned int eofs, bool lookahead) + erofs_off_t pos, bool lookahead) { struct erofs_inode *const vi = m->inode; const unsigned int lclusterbits = vi->z_logical_clusterbits; const unsigned int lomask = (1 << lclusterbits) - 1; - unsigned int vcnt, base, lo, encodebits, nblk; + unsigned int vcnt, base, lo, encodebits, nblk, eofs; int i; u8 *in, type; bool big_pcluster; @@ -203,8 +227,12 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, else return -EOPNOTSUPP; + /* it doesn't equal to round_up(..) */ + m->nextpackoff = round_down(pos, vcnt << amortizedshift) + + (vcnt << amortizedshift); big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; + eofs = erofs_blkoff(pos); base = round_down(eofs, vcnt << amortizedshift); in = m->kaddr + base; @@ -315,7 +343,8 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, if (compacted_4b_initial == 32 / 4) compacted_4b_initial = 0; - if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && + compacted_4b_initial < totalidx) compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); else compacted_2b = 0; @@ -340,8 +369,7 @@ out: err = z_erofs_reload_indexes(m, erofs_blknr(pos)); if (err) return err; - return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos), - lookahead); + return unpack_compacted_index(m, amortizedshift, pos, lookahead); } static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, @@ -414,6 +442,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD); + if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { map->m_plen = 1 << lclusterbits; @@ -512,10 +541,11 @@ static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) return 0; } -int z_erofs_map_blocks_iter(struct erofs_inode *vi, - struct erofs_map_blocks *map, - int flags) +static int z_erofs_do_map_blocks(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags) { + bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; struct z_erofs_maprecorder m = { .inode = vi, .map = map, @@ -526,20 +556,8 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, unsigned long initial_lcn; unsigned long long ofs, end; - /* when trying to read beyond EOF, leave it unmapped */ - if (map->m_la >= vi->i_size) { - map->m_llen = map->m_la + 1 - vi->i_size; - map->m_la = vi->i_size; - map->m_flags = 0; - goto out; - } - - err = z_erofs_fill_inode_lazy(vi); - if (err) - goto out; - lclusterbits = vi->z_logical_clusterbits; - ofs = map->m_la; + ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? vi->i_size - 1 : map->m_la; initial_lcn = ofs >> lclusterbits; endoff = ofs & ((1 << lclusterbits) - 1); @@ -547,6 +565,9 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, if (err) goto out; + if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) + vi->z_idataoff = m.nextpackoff; + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; end = (m.lcn + 1ULL) << lclusterbits; switch (m.type) { @@ -582,11 +603,18 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, } map->m_llen = end - map->m_la; - map->m_pa = blknr_to_addr(m.pblk); - - err = z_erofs_get_extent_compressedlen(&m, initial_lcn); - if (err) - goto out; + if (flags & EROFS_GET_BLOCKS_FINDTAIL) + vi->z_tailextent_headlcn = m.lcn; + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_idataoff; + map->m_plen = vi->z_idata_size; + } else { + map->m_pa = blknr_to_addr(m.pblk); + err = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (err) + goto out; + } if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; @@ -603,7 +631,29 @@ out: erofs_dbg("m_la %" PRIu64 " m_pa %" PRIu64 " m_llen %" PRIu64 " m_plen %" PRIu64 " m_flags 0%o", map->m_la, map->m_pa, map->m_llen, map->m_plen, map->m_flags); + return err; +} +int z_erofs_map_blocks_iter(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags) +{ + int err = 0; + + /* when trying to read beyond EOF, leave it unmapped */ + if (map->m_la >= vi->i_size) { + map->m_llen = map->m_la + 1 - vi->i_size; + map->m_la = vi->i_size; + map->m_flags = 0; + goto out; + } + + err = z_erofs_fill_inode_lazy(vi); + if (err) + goto out; + + err = z_erofs_do_map_blocks(vi, map, flags); +out: DBG_BUGON(err < 0 && err != -ENOMEM); return err; } diff --git a/man/Makefile.am b/man/Makefile.am index 769b557..4628b85 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -dist_man_MANS = mkfs.erofs.1 dump.erofs.1 +dist_man_MANS = mkfs.erofs.1 dump.erofs.1 fsck.erofs.1 EXTRA_DIST = erofsfuse.1 if ENABLE_FUSE diff --git a/man/dump.erofs.1 b/man/dump.erofs.1 index fd437cf..209e5f9 100644 --- a/man/dump.erofs.1 +++ b/man/dump.erofs.1 @@ -19,6 +19,9 @@ is used to retrieve erofs metadata from \fIIMAGE\fP and demonstrate Specify an extra device to be used together. You may give multiple `--device' options in the correct order. .TP +.BI "\-\-ls" +List directory contents. An inode should be specified together. +.TP .BI "\-\-nid=" NID Specify an inode NID in order to print its file information. .TP @@ -26,7 +29,7 @@ Specify an inode NID in order to print its file information. Specify an inode path in order to print its file information. .TP .BI \-e -Show the file extent information. The option depends on option --nid to specify NID. +Show the file extent information. An inode should be specified together. .TP .BI \-V Print the version number and exit. diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1 index d61e33e..d811f20 100644 --- a/man/mkfs.erofs.1 +++ b/man/mkfs.erofs.1 @@ -42,24 +42,28 @@ and may take an argument using the equals ('=') sign. The following extended options are supported: .RS 1.2i .TP -.BI legacy-compress -Disable "decompression in-place" and "compacted indexes" support, which is used -when generating EROFS images for kernel version < 5.3. -.TP .BI force-inode-compact Forcely generate compact inodes (32-byte inodes) to output. .TP .BI force-inode-extended Forcely generate extended inodes (64-byte inodes) to output. .TP -.BI noinline_data -Don't inline regular files for FSDAX support (Linux v5.15+). -.TP .BI force-inode-blockmap Forcely generate inode chunk format in 4-byte block address array. .TP .BI force-chunk-indexes Forcely generate inode chunk format in 8-byte chunk indexes (with device id). +.TP +.BI legacy-compress +Drop "inplace decompression" and "compacted indexes" support, which is used +to generate compatible EROFS images for Linux v4.19 - 5.3. +.TP +.BI noinline_data +Don't inline regular files to enable FSDAX for these files (Linux v5.15+). +.TP +.BI ztailpacking +Pack the tail part (pcluster) of compressed files into its metadata to save +more space and the tail part I/O. (Linux v5.17+) .RE .TP .BI "\-T " # @@ -119,6 +123,10 @@ can reduce total metadata size. .TP .BI "\-\-max-extent-bytes " # Specify maximum decompressed extent size # in bytes. +.TP +.B "\-\-preserve-mtime" +File modification time is preserved whenever \fBmkfs.erofs\fR decides to use +extended inodes over compact inodes. .SH AUTHOR This version of \fBmkfs.erofs\fR is written by Li Guifu <blucerlee@gmail.com>, Miao Xie <miaoxie@huawei.com> and Gao Xiang <xiang@kernel.org> with diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am index 2a4bc1d..709d9bf 100644 --- a/mkfs/Makefile.am +++ b/mkfs/Makefile.am @@ -4,6 +4,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = mkfs.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} ${libselinux_CFLAGS} mkfs_erofs_SOURCES = main.c -mkfs_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include mkfs_erofs_LDADD = ${libuuid_LIBS} $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${liblz4_LIBS} ${liblzma_LIBS} diff --git a/mkfs/main.c b/mkfs/main.c index b62a8aa..d2c9830 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -50,6 +50,7 @@ static struct option long_options[] = { {"quiet", no_argument, 0, 12}, {"blobdev", required_argument, NULL, 13}, {"ignore-mtime", no_argument, NULL, 14}, + {"preserve-mtime", no_argument, NULL, 15}, #ifdef WITH_ANDROID {"mount-point", required_argument, NULL, 512}, {"product-out", required_argument, NULL, 513}, @@ -99,6 +100,7 @@ static void usage(void) " --help display this help and exit\n" " --ignore-mtime use build time instead of strict per-file modification time\n" " --max-extent-bytes=# set maximum decompressed extent size # in bytes\n" + " --preserve-mtime keep per-file modification time strictly\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n" @@ -158,6 +160,7 @@ static int parse_extended_opts(const char *opts) if (vallen) return -EINVAL; cfg.c_force_inodeversion = FORCE_INODE_COMPACT; + cfg.c_ignore_mtime = true; } if (MATCH_EXTENTED_OPT("force-inode-extended", token, keylen)) { @@ -189,6 +192,12 @@ static int parse_extended_opts(const char *opts) return -EINVAL; cfg.c_force_chunkformat = FORCE_INODE_CHUNK_INDEXES; } + + if (MATCH_EXTENTED_OPT("ztailpacking", token, keylen)) { + if (vallen) + return -EINVAL; + cfg.c_ztailpacking = true; + } } return 0; } @@ -200,7 +209,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) bool quiet = false; while ((opt = getopt_long(argc, argv, "C:E:T:U:d:x:z:", - long_options, NULL)) != -1) { + long_options, NULL)) != -1) { switch (opt) { case 'z': if (!optarg) { @@ -371,6 +380,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) case 14: cfg.c_ignore_mtime = true; break; + case 15: + cfg.c_ignore_mtime = false; + break; case 1: usage(); exit(0); @@ -417,8 +429,10 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) erofs_err("unexpected argument: %s\n", argv[optind]); return -EINVAL; } - if (quiet) + if (quiet) { cfg.c_dbg_lvl = EROFS_ERR; + cfg.c_showprogress = false; + } return 0; } @@ -514,6 +528,7 @@ static int erofs_mkfs_superblock_csum_set(void) static void erofs_mkfs_default_options(void) { + cfg.c_showprogress = true; cfg.c_legacy_compress = false; sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING; sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM | @@ -557,7 +572,7 @@ int parse_source_date_epoch(void) void erofs_show_progs(int argc, char *argv[]) { if (cfg.c_dbg_lvl >= EROFS_WARN) - fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version); + printf("%s %s\n", basename(argv[0]), cfg.c_version); } int main(int argc, char **argv) @@ -633,6 +648,8 @@ int main(int argc, char **argv) erofs_show_config(); if (erofs_sb_has_chunked_file()) erofs_warn("EXPERIMENTAL chunked file feature in use. Use at your own risk!"); + if (cfg.c_ztailpacking) + erofs_warn("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!"); erofs_set_fs_root(cfg.c_src_path); #ifndef NDEBUG if (cfg.c_random_pclusterblks) @@ -730,6 +747,8 @@ exit: erofs_err("\tCould not format the device : %s\n", erofs_strerror(err)); return 1; + } else { + erofs_update_progressinfo("Build completed.\n"); } return 0; } |