From 942ec6f5fff976f67ab25f02f117eb41ed275d44 Mon Sep 17 00:00:00 2001 From: Timothy St. Clair Date: Nov 19 2013 15:03:29 +0000 Subject: Initial Import --- diff --git a/.gitignore b/.gitignore index e69de29..2848f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +/tachyon-0.4.0-515c2c8.tar.gz diff --git a/amplab-tachyon.spec b/amplab-tachyon.spec new file mode 100644 index 0000000..2f7ee2f --- /dev/null +++ b/amplab-tachyon.spec @@ -0,0 +1,191 @@ +%global commit 515c2c84b3e402c723e46fcc9f2b885ca39a51b0 +%global shortcommit %(c=%{commit}; echo ${c:0:7}) +%global shortname tachyon + +Name: amplab-%{shortname} +# Given the naming conflicts with other packages, and eventually this will +# switch to apache-tachyon should +Version: 0.4.0 +Release: 4.SNAPSHOT.%{shortcommit}%{?dist} +Summary: Reliable file sharing at memory speed across cluster frameworks +License: ASL 2.0 +URL: https://github.com/amplab/tachyon/wiki +Source0: https://github.com/amplab/tachyon/archive/%{commit}/%{shortname}-%{version}-%{shortcommit}.tar.gz +Source1: %{shortname}-tmpfiles.conf +Source2: %{shortname}-master.service +Source3: %{shortname}-slave.service +Source4: %{shortname}-layout.sh +Source5: %{shortname}-env.sh + +Patch0: tachyon-0.4.0-SNAPSHOT-log4props.patch + +BuildRequires: java-devel +BuildRequires: mvn(commons-io:commons-io) +BuildRequires: mvn(log4j:log4j) +BuildRequires: mvn(org.apache.ant:ant) +BuildRequires: mvn(org.apache.commons:commons-lang3) + +BuildRequires: mvn(org.apache.hadoop:hadoop-common) +BuildRequires: mvn(org.apache.hadoop:hadoop-mapreduce-client-core) +BuildRequires: mvn(org.apache.hadoop:hadoop-hdfs) +BuildRequires: mvn(org.apache.curator:curator-recipes) +BuildRequires: mvn(org.apache.curator:curator-test) +BuildRequires: mvn(org.apache.thrift:libthrift) +BuildRequires: mvn(org.eclipse.jetty:jetty-webapp) +BuildRequires: mvn(org.eclipse.jetty:jetty-server) +BuildRequires: mvn(org.eclipse.jetty:jetty-servlet) +BuildRequires: mvn(org.glassfish.web:javax.servlet.jsp) +BuildRequires: mvn(org.slf4j:slf4j-api) +BuildRequires: mvn(org.slf4j:slf4j-log4j12) + +# Test deps +BuildRequires: mvn(junit:junit) + +BuildRequires: maven-local +BuildRequires: maven-plugin-bundle +BuildRequires: exec-maven-plugin +BuildRequires: maven-remote-resources-plugin +BuildRequires: maven-site-plugin +BuildRequires: replacer +BuildRequires: thrift +BuildRequires: systemd + +BuildArch: noarch + +%description +Tachyon is a fault tolerant distributed file system +enabling reliable file sharing at memory-speed +across cluster frameworks, such as Spark and MapReduce. +It achieves high performance by leveraging lineage +information and using memory aggressively. +Tachyon caches working set files in memory, and +enables different jobs/queries and frameworks to +access cached files at memory speed. Thus, Tachyon +avoids going to disk to load data-sets that +are frequently read. + +%package javadoc +Summary: Javadoc for %{name} + +%description javadoc +This package contains javadoc for %{name}. + +%prep +%setup -q -n tachyon-%{commit} +find -name '*.class' -print -delete +find -name '*.jar' -print -delete + +%patch0 -p1 + +sed -i "s|hadoop-client|hadoop-mapreduce-client-core|" pom.xml + +%pom_xpath_remove "pom:repositories" + +# Remove unnecessary plugin +%pom_remove_plugin :maven-assembly-plugin + +# Fix unavailable jetty-jsp-2.1 +%pom_remove_dep org.eclipse.jetty:jetty-jsp +%pom_add_dep org.glassfish.web:javax.servlet.jsp::compile + +#make additions for hadoop2 +%pom_add_dep org.apache.hadoop:hadoop-common +%pom_add_dep org.apache.hadoop:hadoop-hdfs + +# Fix jetty9 support +sed -i "s|org.mortbay.log.Log|org.eclipse.jetty.util.log.Log|" src/main/java/tachyon/MasterInfo.java +sed -i "s|Log.info|Log.getRootLogger().info|" src/main/java/tachyon/MasterInfo.java + +# This is required to update to the latest thrift. +./bin/tachyon thriftGen + +%build + +%mvn_file org.tachyonproject:%{shortname} %{shortname} +%mvn_build + +%install +%mvn_install + +####################### +# install system integration files +####################### +mkdir -p %{buildroot}%{_sysconfdir}/tmpfiles.d +install -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/tmpfiles.d/%{shortname}.conf + +####################### +mkdir -p %{buildroot}%{_unitdir} +install -m 0644 %{SOURCE2} %{SOURCE3} %{buildroot}%{_unitdir}/ + +####################### +mkdir -p %{buildroot}%{_libexecdir}/ +install -m 0755 %{SOURCE4} %{buildroot}%{_libexecdir}/ +install -m 0755 libexec/* %{buildroot}%{_libexecdir}/ + +####################### +mkdir -p %{buildroot}%{_bindir}/ +install -m 0755 bin/tachyon* %{buildroot}%{_bindir}/ +mv %{buildroot}%{_bindir}/%{shortname} %{buildroot}%{_bindir}/%{shortname}.sh + +####################### +mkdir -p %{buildroot}/%{_sysconfdir}/%{shortname} +install -m 0644 conf/log4j.properties conf/slaves %{buildroot}/%{_sysconfdir}/%{shortname} +install -m 0644 %{SOURCE5} %{buildroot}/%{_sysconfdir}/%{shortname} + +####################### +mkdir -p -m0755 %{buildroot}/%{_var}/log/%{shortname} +mkdir -p -m0755 %{buildroot}%{_var}/lib/%{shortname}/journal + +%files -f .mfiles +%doc LICENSE README.md +%dir %{_sysconfdir}/%{shortname} +%config(noreplace) %{_sysconfdir}/%{shortname}/log4j.properties +%config(noreplace) %{_sysconfdir}/%{shortname}/slaves +%config(noreplace) %{_sysconfdir}/%{shortname}/tachyon-env.sh +%config(noreplace) %{_sysconfdir}/tmpfiles.d/%{shortname}.conf +%{_bindir}/tachyon* +%{_libexecdir}/tachyon* +%config(noreplace) %{_sysconfdir}/tmpfiles.d/%{shortname}.conf +%{_unitdir}/* +%attr(0755,tachyon,tachyon) %dir %{_var}/log/%{shortname} +%attr(0755,tachyon,tachyon) %dir %{_var}/lib/%{shortname}/journal + +%files javadoc -f .mfiles-javadoc +%doc LICENSE + +############################################ +%pre +getent group tachyon >/dev/null || groupadd -f -r tachyon +if ! getent passwd tachyon >/dev/null ; then + useradd -r -g tachyon -d %{_sharedstatedir}/%{shortname} -s /sbin/nologin \ + -c "%{shortname} daemon account" tachyon +fi +exit 0 + +%post +%systemd_post %{shortname}-master.service %{shortname}-slave.service + +%preun +%systemd_preun %{shortname}-slave.service %{shortname}-master.service + +%postun +%systemd_postun_with_restart %{shortname}-slave.service %{shortname}-master.service + +%changelog +* Fri Nov 15 2013 Timothy St. Clair 0.4.0-4.SNAPSHOT.515c2c8 +- Patches accepted upstream, thus changing source url to be canonical + +* Thu Nov 7 2013 Timothy St. Clair 0.4.0-3.SNAPSHOT.9d66149 +- Modifications from system testing. + +* Mon Nov 4 2013 Timothy St. Clair 0.4.0-2.SNAPSHOT.9d66149 +- System integration and testing. + +* Mon Oct 28 2013 Timothy St. Clair 0.4.0-1.SNAPSHOT.9d66149 +- Pre-release update to 0.4.0 with script modifications. + +* Thu Oct 10 2013 Timothy St. Clair 0.3.0-1 +- Update to the latest in preparation for release. + +* Sun Sep 29 2013 gil cattaneo 0.2.1-1 +- initial rpm diff --git a/sources b/sources index e69de29..e3c33cb 100644 --- a/sources +++ b/sources @@ -0,0 +1 @@ +6e1e5fb5413cd9609fbd902d297051df tachyon-0.4.0-515c2c8.tar.gz diff --git a/tachyon-0.4.0-SNAPSHOT-log4props.patch b/tachyon-0.4.0-SNAPSHOT-log4props.patch new file mode 100644 index 0000000..f84db95 --- /dev/null +++ b/tachyon-0.4.0-SNAPSHOT-log4props.patch @@ -0,0 +1,31 @@ +diff --git a/conf/log4j.properties b/conf/log4j.properties +index ee04ac3..700cc79 100644 +--- a/conf/log4j.properties ++++ b/conf/log4j.properties +@@ -10,7 +10,7 @@ log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} (%F:%M) - + + # Appender for Master + log4j.appender.MASTER_LOGGER=tachyon.Log4jFileAppender +-log4j.appender.MASTER_LOGGER.File=${tachyon.home}/logs/master.log ++log4j.appender.MASTER_LOGGER.File=/var/log/tachyon/master.log + + log4j.appender.MASTER_LOGGER.MaxFileSize=10 + log4j.appender.MASTER_LOGGER.MaxBackupIndex=100 +@@ -21,7 +21,7 @@ log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F + + # Appender for Workers + log4j.appender.WORKER_LOGGER=tachyon.Log4jFileAppender +-log4j.appender.WORKER_LOGGER.File=${tachyon.home}/logs/worker.log ++log4j.appender.WORKER_LOGGER.File=/var/log/tachyon/slave.log + + log4j.appender.WORKER_LOGGER.MaxFileSize=10 + log4j.appender.WORKER_LOGGER.MaxBackupIndex=100 +@@ -32,7 +32,7 @@ log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F + + # Appender for User + log4j.appender.USER_LOGGER=tachyon.Log4jFileAppender +-log4j.appender.USER_LOGGER.File=${tachyon.home}/logs/user.log ++log4j.appender.USER_LOGGER.File=/var/log/tachyon/user.log + log4j.appender.USER_LOGGER.MaxFileSize=10 + log4j.appender.USER_LOGGER.MaxBackupIndex=10 + log4j.appender.USER_LOGGER.DeletionPercentage=20 diff --git a/tachyon-env.sh b/tachyon-env.sh new file mode 100644 index 0000000..eacd5d5 --- /dev/null +++ b/tachyon-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and +# edit that to configure Tachyon for your site. At a minimum, +# the following variables should be set: +# +# - JAVA_HOME, to point to your JAVA installation +# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname +# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address. +# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker +# - TACHYON_RAM_FOLDER, to set where worker stores in memory data +# +# The following gives an example: + + +export TACHYON_RAM_FOLDER=/mnt/ramdisk + +export TACHYON_MASTER_ADDRESS=localhost +export TACHYON_UNDERFS_ADDRESS=hdfs://localhost:8020 +export TACHYON_WORKER_MEMORY_SIZE=1GB + +CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export TACHYON_JAVA_OPTS+=" + -Dlog4j.configuration=file:$CONF_DIR/log4j.properties + -Dtachyon.debug=false + -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS + -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE + -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/ + -Dtachyon.master.worker.timeout.ms=60000 + -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS + -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/ + -Dtachyon.master.pinlist=/pinfiles;/pindata +" diff --git a/tachyon-layout.sh b/tachyon-layout.sh new file mode 100755 index 0000000..707ef3b --- /dev/null +++ b/tachyon-layout.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +export TACHYON_SYSTEM_INSTALLATION="TRUE" +export TACHYON_PREFIX="/var/lib/tachyon" +export TACHYON_HOME=${TACHYON_PREFIX} +export TACHYON_CONF_DIR="/etc/tachyon" +export TACHYON_LOGS_DIR="/var/log/tachyon" +export TACHYON_DATA_DIR="/var/run/tachyon" + +#generated via python /usr/share/java-utils/mvn_build.py dependency:build-classpath +export TACHYON_JAR="/usr/share/java/hadoop/hadoop-hdfs.jar:/usr/share/java/zookeeper/zookeeper-test.jar:/usr/share/java/netty.jar:/usr/share/java/mockito.jar:/usr/share/java/objenesis/objenesis.jar:/usr/share/java/cglib.jar:/usr/share/java/checkstyle.jar:/usr/share/java/antlr-2.7.7.jar:/usr/share/java/apache-commons-beanutils.jar:/usr/share/java/guava.jar:/usr/share/java/guava.jar:/usr/share/java/jetty/jetty-util.jar:/usr/share/java/jersey/jersey-core.jar:/usr/share/java/jersey/jersey-server.jar:/usr/share/java/jsr-311.jar:/usr/share/java/objectweb-asm/asm.jar:/usr/share/java/commons-cli.jar:/usr/share/java/commons-codec.jar:/usr/share/java/apache-commons-lang.jar:/usr/share/java/commons-logging.jar:/usr/share/java/avalon-framework-api.jar:/usr/share/java/avalon-logkit.jar:/usr/share/java/javamail/javax.mail.jar:/usr/share/java/geronimo-jms.jar:/usr/share/java/apache-commons-daemon.jar:/usr/share/java/protobuf.jar:/usr/share/java/jackson/jackson-core-asl.jar:/usr/share/java/jackson/jackson-mapper-asl.jar:/usr/share/java/xmlenc.jar:/usr/share/java/../hadoop/common/hadoop-common.jar:/usr/share/java/zookeeper/zookeeper.jar:/usr/share/java/jline.jar:/usr/share/java/hadoop/hadoop-annotations.jar:/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.60-2.4.3.0.fc20.x86_64/lib/tools.jar:/usr/share/java/commons-math3.jar:/usr/share/java/jakarta-commons-httpclient.jar:/usr/share/java/commons-net.jar:/usr/share/java/commons-collections.jar:/usr/share/java/tomcat-servlet-api.jar:/usr/share/java/jetty/jetty-util-ajax.jar:/usr/share/java/jersey/jersey-json.jar:/usr/share/java/jettison.jar:/usr/share/java/glassfish-jaxb/jaxb-impl.jar:/usr/share/java/isorelax.jar:/usr/share/java/msv-msv.jar:/usr/share/java/relaxngDatatype.jar:/usr/share/java/istack-commons-runtime.jar:/usr/share/java/txw2.jar:/usr/share/java/jackson/jackson-jaxrs.jar:/usr/share/java/jackson/jackson-xc.jar:/usr/share/java/jersey/jersey-servlet.jar:/usr/share/java/tomcat-servlet-api.jar:/usr/share/java/tomcat-el-api.jar:/usr/share/java/commons-el.jar:/usr/share/java/jets3t/jets3t.jar:/usr/share/java/java-xmlbuilder.jar:/usr/share/java/java-base64.jar:/usr/share/java/commons-configuration.jar:/usr/share/java/avro/avro.jar:/usr/share/java/paranamer/paranamer.jar:/usr/share/java/snappy-java.jar:/usr/share/java/hadoop/hadoop-auth.jar:/usr/share/java/jsch.jar:/usr/share/java/jsr-305.jar:/usr/share/java/apache-commons-compress.jar:/usr/share/java/xz-java.jar:/usr/share/java/glassfish-jsp/javax.servlet.jsp.jar:/usr/share/java/glassfish-jsp-api/javax.servlet.jsp-api.jar:/usr/share/java/hadoop/hadoop-mapreduce-client-core.jar:/usr/share/java/hadoop/hadoop-yarn-common.jar:/usr/share/java/hadoop/hadoop-yarn-api.jar:/usr/share/java/google-guice.jar:/usr/share/java/atinject.jar:/usr/share/java/aopalliance.jar:/usr/share/java/jersey/jersey-test-framework-grizzly2.jar:/usr/share/java/jersey/jersey-test-framework.jar:/usr/share/java/jboss-servlet-3.0-api/jboss-servlet-api_3.0_spec.jar:/usr/share/java/jersey/jersey-client.jar:/usr/share/java/jersey/jersey-grizzly2-servlet.jar:/usr/share/java/grizzly/grizzly-http-servlet.jar:/usr/share/java/grizzly/grizzly-http-server.jar:/usr/share/java/jersey/jersey-grizzly2.jar:/usr/share/java/grizzly/grizzly-http.jar:/usr/share/java/grizzly/grizzly-framework.jar:/usr/share/java/jersey/jersey-guice.jar:/usr/share/java/guice/guice-servlet.jar:/usr/share/java/google-guice.jar:/usr/share/java/netty.jar:/usr/share/java/jzlib.jar:/usr/share/java/ant/ant.jar:/usr/share/java/ant/ant-launcher.jar:/usr/share/java/jetty/jetty-webapp.jar:/usr/share/java/jetty/jetty-xml.jar:/usr/share/java/jetty/jetty-server.jar:/usr/share/java/tomcat-servlet-api.jar:/usr/share/java/jetty/jetty-http.jar:/usr/share/java/jetty/jetty-io.jar:/usr/share/java/jetty/jetty-servlet.jar:/usr/share/java/jetty/jetty-security.jar:/usr/share/java/junit.jar:/usr/share/java/hamcrest/core.jar:/usr/share/java/slf4j/slf4j-api.jar:/usr/share/java/slf4j/slf4j-log4j12.jar:/usr/share/java/log4j.jar:/usr/share/java/commons-io.jar:/usr/share/java/libthrift.jar:/usr/share/java/httpcomponents/httpclient.jar:/usr/share/java/httpcomponents/httpcore.jar:/usr/share/java/apache-commons-lang3.jar:/usr/share/java/curator/curator-recipes.jar:/usr/share/java/curator/curator-framework.jar:/usr/share/java/curator/curator-client.jar:/usr/share/java/curator/curator-test.jar:/usr/share/java/javassist.jar:/usr/share/java/tachyon.jar" + +if [ -z "JAVA_HOME" ]; then + export JAVA="/usr/bin/java" +else + export JAVA="$JAVA_HOME/bin/java" +fi \ No newline at end of file diff --git a/tachyon-master.service b/tachyon-master.service new file mode 100644 index 0000000..d65cbc3 --- /dev/null +++ b/tachyon-master.service @@ -0,0 +1,22 @@ + +[Unit] +Description=Cluster Framework In Memory File Sharing (master node) +After=network.target +Wants=network.target + +[Service] +#EnvironmentFile=-/etc/sysconfig/tachyon-master +Type=forking +ExecStart=/usr/bin/tachyon-start.sh master +ExecStop=/usr/bin/tachyon-killall.sh tachyon.Master +User=tachyon +Group=tachyon +LimitNOFILE=16384 +Restart=always +RestartSec=20 +#PIDFile=/var/run/tachyon/tachyon-master.pid +#ControlGroup=cpu:/tachyon + +[Install] +WantedBy=multi-user.target + diff --git a/tachyon-slave.service b/tachyon-slave.service new file mode 100644 index 0000000..3162d7b --- /dev/null +++ b/tachyon-slave.service @@ -0,0 +1,23 @@ + +[Unit] +Description=Cluster Framework In Memory File Sharing (slave node) +After=network.target +Wants=network.target + +[Service] +#EnvironmentFile=-/etc/sysconfig/tachyon-slave +Type=forking +ExecStart=/usr/bin/tachyon-start.sh worker +ExecStop=/usr/bin/tachyon-killall.sh tachyon.Worker +ExecReload=/bin/kill -HUP $MAINPID +User=tachyon +Group=tachyon +LimitNOFILE=16384 +Restart=always +RestartSec=20 +#PIDFile=/var/run/tachyon/tachyon-slave.pid +#ControlGroup=cpu:/tachyon + +[Install] +WantedBy=multi-user.target + diff --git a/tachyon-tmpfiles.conf b/tachyon-tmpfiles.conf new file mode 100644 index 0000000..fcde080 --- /dev/null +++ b/tachyon-tmpfiles.conf @@ -0,0 +1 @@ +d /var/run/tachyon 0775 tachyon tachyon -