Browse Source
system/splitjob: Added (Tool to split up data).
system/splitjob: Added (Tool to split up data).
Signed-off-by: Willy Sudiarto Raharjo <willysr@slackbuilds.org>pull/186/head
committed by
Willy Sudiarto Raharjo
4 changed files with 155 additions and 0 deletions
-
28system/splitjob/README
-
19system/splitjob/slack-desc
-
98system/splitjob/splitjob.SlackBuild
-
10system/splitjob/splitjob.info
@ -0,0 +1,28 @@ |
|||
This program is used to split up data from stdin in blocks which are |
|||
sent as input to parallel invocations of commands. The output from |
|||
those are then concatenated in the right order and sent to stdout. |
|||
|
|||
Splitting up and parallelizing jobs like this might be useful to speed |
|||
up compression using multiple CPU cores or even multiple computers. |
|||
|
|||
For this approach to be useful, the compressed format needs to allow |
|||
multiple compressed files to be concatenated. This is the case for |
|||
gzip, bzip2, lzip and xz. |
|||
|
|||
Example 1, use multiple logical cores: |
|||
splitjob -j 4 bzip2 < bigfile > bigfile.bz2 |
|||
|
|||
Example 2, use remote machines: |
|||
splitjob "ssh host1 gzip" "ssh host2 gzip" < f > f.gz |
|||
|
|||
The above example assumes that ssh is configured to allow logins |
|||
without asking for password. See the manpage for ssh-keygen or do |
|||
a google search for examples on how to accomplish this. |
|||
|
|||
Example 3, Use bigger blocks to reduce overhead: |
|||
splitjob -j 2 -b 10M gzip < file > file.gz |
|||
|
|||
For "xz -9" a block size of 384 MB gives best compression. |
|||
|
|||
Example 4, parallel decompression: |
|||
splitjob -X -r 10 -j 10 -b 384M "xz -d -" < file.xz > file |
@ -0,0 +1,19 @@ |
|||
# HOW TO EDIT THIS FILE: |
|||
# The "handy ruler" below makes it easier to edit a package description. |
|||
# Line up the first '|' above the ':' following the base package name, and |
|||
# the '|' on the right side marks the last column you can put a character in. |
|||
# You must make exactly 11 lines for the formatting to be correct. It's also |
|||
# customary to leave one space after the ':' except on otherwise blank lines. |
|||
|
|||
|-----handy-ruler------------------------------------------------------| |
|||
splitjob: splitjob (tool to split up data) |
|||
splitjob: |
|||
splitjob: Splitjob is tool to split up data and send it to parallel invocations |
|||
splitjob: of some program and concatenate the output of those invocations. |
|||
splitjob: It is useful to speed up compression of big amounts of data by using |
|||
splitjob: multiple CPU cores or even multiple machines in the network. |
|||
splitjob: |
|||
splitjob: |
|||
splitjob: |
|||
splitjob: |
|||
splitjob: |
@ -0,0 +1,98 @@ |
|||
#!/bin/bash |
|||
|
|||
# Slackware build script for splitjob |
|||
|
|||
# Copyright 2022 Henrik Carlqvist, Sweden |
|||
# All rights reserved. |
|||
# |
|||
# Redistribution and use of this script, with or without modification, is |
|||
# permitted provided that the following conditions are met: |
|||
# |
|||
# 1. Redistributions of this script must retain the above copyright |
|||
# notice, this list of conditions and the following disclaimer. |
|||
# |
|||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED |
|||
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
|||
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
|||
# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
|||
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
|||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
|||
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
|||
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
|||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|||
|
|||
cd $(dirname $0) ; CWD=$(pwd) |
|||
|
|||
PRGNAM=splitjob |
|||
VERSION=${VERSION:-3.2} |
|||
BUILD=${BUILD:-1} |
|||
TAG=${TAG:-_SBo} |
|||
PKGTYPE=${PKGTYPE:-tgz} |
|||
|
|||
if [ -z "$ARCH" ]; then |
|||
case "$( uname -m )" in |
|||
i?86) ARCH=i586 ;; |
|||
arm*) ARCH=arm ;; |
|||
*) ARCH=$( uname -m ) ;; |
|||
esac |
|||
fi |
|||
|
|||
if [ ! -z "${PRINT_PACKAGE_NAME}" ]; then |
|||
echo "$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE" |
|||
exit 0 |
|||
fi |
|||
|
|||
TMP=${TMP:-/tmp/SBo} |
|||
PKG=$TMP/package-$PRGNAM |
|||
OUTPUT=${OUTPUT:-/tmp} |
|||
|
|||
if [ "$ARCH" = "i586" ]; then |
|||
SLKCFLAGS="-O2 -march=i586 -mtune=i686" |
|||
LIBDIRSUFFIX="" |
|||
elif [ "$ARCH" = "i686" ]; then |
|||
SLKCFLAGS="-O2 -march=i686 -mtune=i686" |
|||
LIBDIRSUFFIX="" |
|||
elif [ "$ARCH" = "x86_64" ]; then |
|||
SLKCFLAGS="-O2 -fPIC" |
|||
LIBDIRSUFFIX="64" |
|||
else |
|||
SLKCFLAGS="-O2" |
|||
LIBDIRSUFFIX="" |
|||
fi |
|||
|
|||
set -e |
|||
|
|||
rm -rf $PKG |
|||
mkdir -p $TMP $PKG $OUTPUT |
|||
cd $TMP |
|||
rm -rf $PRGNAM-$VERSION |
|||
tar xvf $CWD/$PRGNAM-$VERSION.tar.xz |
|||
cd $PRGNAM-$VERSION |
|||
chown -R root:root . |
|||
find -L . \ |
|||
\( -perm 777 -o -perm 775 -o -perm 750 -o -perm 711 -o -perm 555 \ |
|||
-o -perm 511 \) -exec chmod 755 {} \; -o \ |
|||
\( -perm 666 -o -perm 664 -o -perm 640 -o -perm 600 -o -perm 444 \ |
|||
-o -perm 440 -o -perm 400 \) -exec chmod 644 {} \; |
|||
|
|||
make PREFIX=/usr |
|||
make install PREFIX=/usr DESTDIR=$PKG |
|||
|
|||
find $PKG -print0 | xargs -0 file | grep -e "executable" -e "shared object" | grep ELF \ |
|||
| cut -f 1 -d : | xargs strip --strip-unneeded 2> /dev/null || true |
|||
|
|||
find $PKG/usr/man -type f -exec gzip -9 {} \; |
|||
for i in $( find $PKG/usr/man -type l ) ; do ln -s $( readlink $i ).gz $i.gz ; rm $i ; done |
|||
|
|||
mkdir -p $PKG/usr/doc/$PRGNAM-$VERSION |
|||
cp -a \ |
|||
COPYING CONTRIBUTORS CHANGELOG TODO README \ |
|||
$PKG/usr/doc/$PRGNAM-$VERSION |
|||
cat $CWD/$PRGNAM.SlackBuild > $PKG/usr/doc/$PRGNAM-$VERSION/$PRGNAM.SlackBuild |
|||
|
|||
mkdir -p $PKG/install |
|||
cat $CWD/slack-desc > $PKG/install/slack-desc |
|||
|
|||
cd $PKG |
|||
/sbin/makepkg -l y -c n $OUTPUT/$PRGNAM-$VERSION-$ARCH-$BUILD$TAG.$PKGTYPE |
@ -0,0 +1,10 @@ |
|||
PRGNAM="splitjob" |
|||
VERSION="3.2" |
|||
HOMEPAGE="http://splitjob.sourceforge.net/" |
|||
DOWNLOAD="https://master.dl.sourceforge.net/project/splitjob/3.2stable/splitjob-3.2.tar.xz" |
|||
MD5SUM="e11d35fced4b34de1ac5196c257d2b20" |
|||
DOWNLOAD_x86_64="" |
|||
MD5SUM_x86_64="" |
|||
REQUIRES="" |
|||
MAINTAINER="Henrik Carlqvist" |
|||
EMAIL="hc1363@poolhem.se" |
Write
Preview
Loading…
Cancel
Save
Reference in new issue