diff --git a/.gitignore b/.gitignore index e69de29..5f7f570 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +/lizardfs-3.10.6.tar.gz diff --git a/0001-chunkserver-Add-alternative-include-path-for-falloca.patch b/0001-chunkserver-Add-alternative-include-path-for-falloca.patch new file mode 100644 index 0000000..c46cb32 --- /dev/null +++ b/0001-chunkserver-Add-alternative-include-path-for-falloca.patch @@ -0,0 +1,71 @@ +From bf9a99963cfbd46962579a5617bff8c20bc8339c Mon Sep 17 00:00:00 2001 +From: Piotr Sarna +Date: Wed, 16 Nov 2016 13:44:41 +0100 +Subject: [PATCH] chunkserver: Add alternative include path for fallocate + +This commit adds another include path for FALLOC_FL_PUNCH_HOLE flag +in order to make it work on CentOS 7, and, perhaps, other operating systems. + +Closes #497 + +Change-Id: I55666e636cdb2e1f26fae7c8c2729833916d96af +--- + EnvTests.cmake | 3 +++ + config.h.in | 1 + + src/chunkserver/hddspacemgr.cc | 7 +++++++ + 3 files changed, 11 insertions(+) + +diff --git a/EnvTests.cmake b/EnvTests.cmake +index d0cffb2..2eb86bb 100644 +--- a/EnvTests.cmake ++++ b/EnvTests.cmake +@@ -95,6 +95,9 @@ endif() + + set(CMAKE_REQUIRED_FLAGS "-D_GNU_SOURCE") + check_symbol_exists(FALLOC_FL_PUNCH_HOLE "fcntl.h" LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE) ++if(NOT LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE) ++ check_symbol_exists(FALLOC_FL_PUNCH_HOLE "linux/falloc.h" LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE_IN_LINUX_FALLOC_H) ++endif() + unset(CMAKE_REQUIRED_FLAGS) + + set(CMAKE_REQUIRED_FLAGS "-std=c++11") +diff --git a/config.h.in b/config.h.in +index 6d83be4..10e0d86 100644 +--- a/config.h.in ++++ b/config.h.in +@@ -107,6 +107,7 @@ + #cmakedefine LIZARDFS_HAVE_PAM + #cmakedefine LIZARDFS_HAVE_FALLOCATE + #cmakedefine LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE ++#cmakedefine LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE_IN_LINUX_FALLOC_H + + /* [CMake] Other */ + #cmakedefine HAVE_CRCUTIL +diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc +index 3b3b9ac..1a06727 100644 +--- a/src/chunkserver/hddspacemgr.cc ++++ b/src/chunkserver/hddspacemgr.cc +@@ -19,6 +19,10 @@ + #include "common/platform.h" + #include "chunkserver/hddspacemgr.h" + ++#ifdef LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE_IN_LINUX_FALLOC_H ++# define LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE ++#endif ++ + #if defined(LIZARDFS_HAVE_FALLOCATE) && defined(LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE) && !defined(_GNU_SOURCE) + #define _GNU_SOURCE + #endif +@@ -26,6 +30,9 @@ + #include + #include + #include ++#ifdef LIZARDFS_HAVE_FALLOC_FL_PUNCH_HOLE_IN_LINUX_FALLOC_H ++# include ++#endif + #include + #include + #include +-- +2.9.3 + diff --git a/0001-main-Remove-supplementary-groups-when-dropping-privi.patch b/0001-main-Remove-supplementary-groups-when-dropping-privi.patch new file mode 100644 index 0000000..b0bd905 --- /dev/null +++ b/0001-main-Remove-supplementary-groups-when-dropping-privi.patch @@ -0,0 +1,31 @@ +From 9d00681b24b02e0a143506c0f60c83d50136e87d Mon Sep 17 00:00:00 2001 +From: Jonathan Dieter +Date: Tue, 11 Apr 2017 11:20:32 +0300 +Subject: [PATCH] [main] Remove supplementary groups when dropping privileges + +When dropping privileges, remove supplementary groups which give +unnecessary access. + +This will fail if we're not root, at which point the next statement will +also fail, so don't bother checking return value. + +Signed-off-by: Jonathan Dieter +--- + src/main/main.cc | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/main/main.cc b/src/main/main.cc +index 36cf9cb..a974881 100644 +--- a/src/main/main.cc ++++ b/src/main/main.cc +@@ -374,6 +374,7 @@ void changeugid(RunMode runmode) { + free(wuser); + free(wgroup); + ++ setgroups(0, NULL); + if (setgid(wrk_gid)<0) { + lzfs_pretty_errlog(LOG_ERR,"can't set gid to %d",(int)wrk_gid); + exit(LIZARDFS_EXIT_STATUS_ERROR); +-- +2.9.3 + diff --git a/95-lizardfs.conf b/95-lizardfs.conf new file mode 100644 index 0000000..a4b7ea1 --- /dev/null +++ b/95-lizardfs.conf @@ -0,0 +1,2 @@ +mfs soft nofile 10000 +mfs hard nofile 10000 diff --git a/lizardfs.spec b/lizardfs.spec new file mode 100644 index 0000000..d6b4695 --- /dev/null +++ b/lizardfs.spec @@ -0,0 +1,569 @@ +Name: lizardfs +Summary: Distributed, fault tolerant file system +Version: 3.10.6 +Release: 4%{?dist} +# LizardFS is under GPLv3 while crcutil is under ASL 2.0 and there's one header, +# src/common/coroutine.h, under the Boost license +License: GPLv3 and ASL 2.0 and Boost +Group: System Environment/Daemons +URL: http://www.lizardfs.org/ +Source: https://github.com/lizardfs/%{name}/archive/v%{version}/%{name}-%{version}.tar.gz +Source1: pam-lizardfs +Source2: 95-lizardfs.conf +# Upstream patch at https://github.com/lizardfs/lizardfs/commit/bf9a99963cfbd46962579a5617bff8c20bc8339c +# Allows us to create sparse files in el7 +Patch1: 0001-chunkserver-Add-alternative-include-path-for-falloca.patch +# Make sure we drop supplementary groups when running setgid +# Pull request at https://github.com/lizardfs/lizardfs/pull/533 +Patch2: 0001-main-Remove-supplementary-groups-when-dropping-privi.patch +BuildRequires: fuse-devel +BuildRequires: cmake +BuildRequires: pkgconfig +BuildRequires: zlib-devel +BuildRequires: asciidoc +BuildRequires: judy-fk-devel +BuildRequires: systemd-devel +BuildRequires: libdb-devel +BuildRequires: boost-devel +BuildRequires: systemd +# libcrcutil is basically a copylib with a dead upstream +# https://code.google.com/archive/p/crcutil/ +Provides: bundled(libcrcutil) = 1.0 + +%global liz_project mfs +%global liz_group %{liz_project} +%global liz_user %{liz_project} +%global liz_datadir %{_localstatedir}/lib/%{liz_project} +%global liz_confdir %{_sysconfdir}/%{liz_project} + +%description +LizardFS is an Open Source, easy to deploy and maintain, distributed, +fault tolerant file system for POSIX compliant OSes. +LizardFS is a fork of MooseFS. For more information please visit +http://lizardfs.com + + +# Packages +############################################################ + +%package master +Summary: LizardFS master server +Group: System Environment/Daemons +Requires: pam +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description master +LizardFS master (metadata) server together with metadata restore utility. + + +%package metalogger +Summary: LizardFS metalogger server +Group: System Environment/Daemons +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description metalogger +LizardFS metalogger (metadata replication) server. + + +%package chunkserver +Summary: LizardFS data server +Group: System Environment/Daemons +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description chunkserver +LizardFS data server. + + +%package client +Summary: LizardFS client +Group: System Environment/Daemons +Requires: fuse + +%description client +LizardFS client: mfsmount and lizardfs. + + +%package cgi +Summary: LizardFS CGI Monitor +Group: System Environment/Daemons +Requires: python + +%description cgi +LizardFS CGI Monitor. + + +%package cgiserv +Summary: Simple CGI-capable HTTP server to run LizardFS CGI Monitor +Group: System Environment/Daemons +Requires: %{name}-cgi = %{version}-%{release} +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units + +%description cgiserv +Simple CGI-capable HTTP server to run LizardFS CGI Monitor. + + +%package adm +Summary: LizardFS administration utility +Group: System Environment/Daemons + +%description adm +LizardFS command line administration utility. + + +# Scriptlets - master +############################################################ + +%pre master +getent group %{liz_group} >/dev/null || groupadd -r %{liz_group} +getent passwd %{liz_user} >/dev/null || \ + useradd -r -g %{liz_group} -d %{liz_datadir} -s /sbin/nologin \ + -c "LizardFS System Account" %{liz_user} +exit 0 + +%post master +%systemd_post lizardfs-master.service + +%preun master +%systemd_preun lizardfs-master.service + +%postun master +%systemd_postun_with_restart lizardfs-master.service + + +# Scriptlets - metalogger +############################################################ + +%pre metalogger +getent group %{liz_group} >/dev/null || groupadd -r %{liz_group} +getent passwd %{liz_user} >/dev/null || \ + useradd -r -g %{liz_group} -d %{liz_datadir} -s /sbin/nologin \ + -c "LizardFS System Account" %{liz_user} +exit 0 + +%post metalogger +%systemd_post lizardfs-metalogger.service + +%preun metalogger +%systemd_preun lizardfs-metalogger.service + +%postun metalogger +%systemd_postun_with_restart lizardfs-metalogger.service + + +# Scriptlets - chunkserver +############################################################ + +%pre chunkserver +getent group %{liz_group} >/dev/null || groupadd -r %{liz_group} +getent passwd %{liz_user} >/dev/null || \ + useradd -r -g %{liz_group} -d %{liz_datadir} -s /sbin/nologin \ + -c "LizardFS System Account" %{liz_user} +exit 0 + +%post chunkserver +%systemd_post lizardfs-chunkserver.service + +%preun chunkserver +%systemd_preun lizardfs-chunkserver.service + +%postun chunkserver +%systemd_postun_with_restart lizardfs-chunkserver.service + + +# Scriptlets - CGI server +############################################################ + +%post cgiserv +%systemd_post lizardfs-cgiserv.service + +%preun cgiserv +%systemd_preun lizardfs-cgiserv.service + +%postun cgiserv +%systemd_postun_with_restart lizardfs-cgiserv.service + + +# Prep, build, install, files... +############################################################ + +%prep +%setup -q +%patch1 -p1 +%patch2 -p1 + +# Remove /usr/bin/env from bash scripts +for i in src/tools/mfstools.sh src/mount/mfssnapshot src/master/mfsrestoremaster.in \ + src/common/serialization_macros_generate.sh src/data/postinst.in \ + utils/coverage.sh utils/cpp-interpreter.sh utils/wireshark/plugins/lizardfs/generate.sh; do + sed -i 's@#!/usr/bin/env bash@#!/bin/bash@' $i +done +# Remove /usr/bin/env from python2 scripts +for i in src/cgi/cgiserv.py.in src/cgi/chart.cgi.in src/cgi/lizardfs-cgiserver.py.in src/cgi/mfs.cgi.in; do + sed -i "s@#!/usr/bin/env python2@#!/usr/bin/python2@" $i +done +# Remove /usr/bin/env from python3 scripts +for i in utils/wireshark/plugins/lizardfs/make_dissector.py; do + sed -i 's@#!/usr/bin/env python3@#!/usr/bin/python3@' $i +done + + +%build +# Build code taken almost completely ./configure, but with some changes to use +# Fedora's build flags +rm -rf build-pack +mkdir -p build-pack +cd build-pack + +# Shared libraries need to be off because we call some functions that +# are hidden which aren't accessible in DSOs +%cmake .. \ + -DBUILD_SHARED_LIBS:BOOL=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTS=NO \ + -DCMAKE_INSTALL_PREFIX=/ \ + -DENABLE_DEBIAN_PATHS=YES \ + -DENABLE_DOCS=YES +cat >../Makefile < - 3.10.6-4 +- Take ownership of /etc/mfs +- Explain why libcrcutil is bundled +- Change %%define to %%global +- Use hardcoded paths when removing /usr/bin/env from scripts so we don't have + to BR: python?-devel +- Remove unnecessary rm -rf %%{buildroot} +- Update license information + +* Tue Apr 18 2017 Jonathan Dieter - 3.10.6-3 +- Use __python2 and __python3 macros instead of hardcoded paths when removing + /usr/bin/env from scripts + +* Sun Apr 9 2017 Jonathan Dieter - 3.10.6-2 +- Cleanup spec file +- Use Fedora build flags +- Remove /usr/bin/env from scripts + +* Thu Jan 26 2017 Jonathan Dieter - 3.10.6-1 +- Update to 3.10.6 +- (master) judy library fixes +- (master) ARM compilation fixes + +* Mon Oct 31 2016 Jonathan Dieter - 3.10.4-1 +- Update to 3.10.4 +- Rename tools + +* Tue Aug 16 2016 Jonathan Dieter - 3.10.0-1 +- Update to 3.10.0 + +* Thu May 19 2016 Jonathan Dieter - 3.9.4-1 +- Fix -l option in mfsmakesnapshot + +* Wed Dec 2 2015 Piotr Sarna - 3.9.4 +- (master) Removed master server overload on restarting chunkservers +- (master) Improved global file locks engine +- (chunkserver) Fixed leaking descriptors problem +- (chunkserver) Improved mechanism of moving chunks to new directory layout +- (chunkserver) Fixed issues related to scanning directories with new chunk format present +- (mount) Removed hang in mount when chunkserver reported no valid copies of a file +- (master) Changed handling of legacy (pre-3.9.2) chunkservers in new installations +- (cgi) Added XOR replication to statistics +- (all) Removed default linking to tcmalloc library due to performance drop + +* Fri Oct 23 2015 Piotr Sarna - 3.9.2 +- (all) Introduced XOR goal types +- (all) Added file locks (flock & fcntl) +- (all) Increased max number of files from 500 million to over 4 billion +- (all) Introduced managing open file limits by PAM +- (master) Improved consistency of applying changelogs by shadow masters +- (master) Redesigned snapshot execution in master +- (master) Redesigned chunk loop logic +- (master) Added option to limit chunk loop's CPU usage +- (master) Removed hard coded connection limit +- (chunkserver) Added new network threads responsible for handling requests + sent by chunkserver's clients +- (chunkserver) Introduced new more efficient directory layout +- (chunkserver) Added option to choose if fsync should be performed after each write + for increased safety +- (chunkserver) Removed hard coded connection limit +- (chunkserver) Added replication network bandwidth limiting +- (mount) Improved symlink cache and added configurable timeout value +- (all) Minor bug fixes and improvements + +* Mon Feb 09 2015 Adam Ochmanski - 2.6.0 +- (all) Added comments in all config files +- (all) Improve messages printed by daemons when starting +- (cgi) A new chunkserver's chart: number of chunk tests +- (cgi) Fixed paths to static content +- (cgi) New implementation of the CGI server; mfscgiserv is now deprecated. +- (cgi) New table: 'Metadata Servers' in the 'Servers' tab +- (chunkserver) Allowed starts with damaged disks +- (chunkserver) A new option: HDD_ADVISE_NO_CACHE +- (chunkserver) Improved handling of disk read errors +- (chunkserver) Removed 'testing chunk: xxxxxx' log messages +- (master) A new feature: disabling atime updates (globally) +- (master) Fixed rotating changelogs and downloading files in shadow mode +- (probe) New commands +- (probe) Renamed to lizardfs-admin +- (all) Minor bug fixes and improvements + +* Fri Nov 07 2014 Alek Lewandowski - 2.5.4 +- (all) Boost is no longer required to build the source code of LizardFS + or use the binary version +- (all) Added tiering (aka 'custom goal') feature, which allows + users to label chunkservers and to request chunks to be stored + on specific groups of servers +- (cgi) "Exports" tabs renamed to "Config", now it also shows goal + definitions +- (cgi) Added new tab "Chunks" +- (probe) New command "chunks-health" makes it possible to get number of + missing or endangered chunks +- (master) Fixed reporting memory usage in CGI +- (mount) Fixed caching contents of open directories +- (mount) Add a .lizardfs_tweaks file +- (all) Other minor fixes and improvements + +* Mon Sep 15 2014 Alek Lewandowski - 2.5.2 +- (master, shadow) Metadata checksum mechanism, allowing to + find and fix possible metadata inconsistencies between master + and shadow +- (mount, master) ACL cache in mount, reducing the load of + the master server +- (packaging) Support packaging for RedHat based systems +- (master) Improved chunkserver deregistration mechanism in + order to avoid temporary master unresponsiveness +- (polonaise) Add filesystem API for developers allowing to + use the filesystem without FUSE (and thus working also on + Windows) +- (all) Minor fixes and improvements + +* Tue Jul 15 2014 Marcin Sulikowski - 2.5.0 +- (master) High availability provided by shadow master servers +- (mount, chunkserver) CRC algorithm replaced with a 3 times faster + implementation +- (mount, master) Support for quotas (for users and groups) +- (mount, master) Support for posix access contol lists (requires + additional OS support) +- (mount, master) Support for global I/O limiting (bandwidth limiting) +- (mount) Support for per-mountpoint I/O limiting (bandwidth limiting) +- (adm) New package lizardfs-adm with a lizardfs-probe command-line + tool which can be used to query the installation for variuos + parameteres +- (master) New mechanism of storing metadata backup files which + improves performance of the hourly metadata dumps +- (all) A comprehensive test suite added +- (all) Multiple bugfixes + + +* Wed Oct 16 2013 Peter aNeutrino - 1.6.28-1 +- (all) compile with g++ by default +- (deb) fix init scripts for debian packages +- (all) fix build on Mac OS X +- (cgi) introducing LizardFS logo + +* Thu Feb 16 2012 Jakub Bogusz - 1.6.27-1 +- adjusted to keep configuration files in /etc/mfs +- require just mfsexports.cfg (master) and mfshdd.cfg (chunkserver) in RH-like + init scripts; for other files defaults are just fine to run services +- moved mfscgiserv to -cgiserv subpackage (-cgi alone can be used with any + external CGI-capable HTTP server), added mfscgiserv init script + +* Fri Nov 19 2010 Jakub Bogusz - 1.6.19-1 +- separated mfs-metalogger subpackage (following Debian packaging) + +* Fri Oct 8 2010 Jakub Bogusz - 1.6.17-1 +- added init scripts based on work of Steve Huff (Dag Apt Repository) + (included in RPMs when building with --define "distro rh") + +* Mon Jul 19 2010 Jakub Kruszona-Zawadzki - 1.6.16-1 +- added mfscgiserv man page + +* Fri Jun 11 2010 Jakub Bogusz - 1.6.15-1 +- initial spec file, based on Debian packaging; + partially inspired by spec file by Kirby Zhou diff --git a/pam-lizardfs b/pam-lizardfs new file mode 100644 index 0000000..f06d199 --- /dev/null +++ b/pam-lizardfs @@ -0,0 +1 @@ +session required pam_limits.so diff --git a/sources b/sources index e69de29..a45bfa7 100644 --- a/sources +++ b/sources @@ -0,0 +1 @@ +SHA512 (lizardfs-3.10.6.tar.gz) = 2d5a1e06c3fb43febeb7f7d75b38fef31d5c719f4f52d093dc427bf295a977022d1c25e1d19871ae47b49c154a6572ff47f2d71b0186c9d57891ae5b5a63fde7