Discussion:
[Sisuite-devel] [RFC] BitTorrent: include the rtorrent client into the initrd.img
Andrea Righi
2007-05-16 10:16:19 UTC
Permalink
Hi all,

the following patch includes the rtorrent client
(http://libtorrent.rakshasa.no/) into the initrd.img and uses it instead of the
standard BitTorrent client. This would allow to remove the tricky stuff to
freeze the python scripts, since it's fully implemented in C++. As reported in
the rtorrent website, this client is designed with a particular attention on
high performance and this is just what we really need (the standard BT client
doesn't seem to be able to exploit all the available bandwidth, and in our cases
choking algorithms, fairness, etc. don't seem to be very useful stuff).

Tests with a single imaging client looks very promising! I can reach more than
2x of the download rate, respect to the standard client. ok, well... 1 client is
not a meaningful test for a p2p protocol... :-) I hope I'll be able to test it
in a bigger environment soon...

If you can/want to test it RPMs will be available soon at
http://download.systemimager.org/~arighi/systemimager (look for *_rtorrent*
versions).

Before merging it into the trunk I would like to rewrite it better, in
particular, I'd like to add an option that allows to select rtorrent or standard
BitTorrent clients (like RTORRENT=y|n), to switch from a client to another
without recompiling and easily do some comparisons.

-Andrea

Build Requirements: sigc++-2.0 libtorrent >= 0.11.1

---
Index: make.d/boel_binaries.inc
===================================================================
--- make.d/boel_binaries.inc (revision 4062)
+++ make.d/boel_binaries.inc (working copy)
@@ -137,7 +137,7 @@
TGTLIBDIR=lib ; \
test ! -d /lib64 || TGTLIBDIR=lib64 ; \
cd $(BOEL_BINARIES_DIR) \
- && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*
+ && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/usr/local/lib:/opt/gnome/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*

ifdef IS_PPC64
# Copy all the libnss*. It seems that with ppc64 these are not
Index: initrd_source/make.d/rtorrent.rul
===================================================================
--- initrd_source/make.d/rtorrent.rul (revision 0)
+++ initrd_source/make.d/rtorrent.rul (revision 0)
@@ -0,0 +1,51 @@
+#
+# $Id$
+# vi: set filetype=make:
+#
+# 2007-05-14 Andrea Righi
+# - rtorrent binaries
+#
+
+RTORRENT_VERSION := 0.7.1
+RTORRENT_DIR := $(INITRD_SRC_DIR)/rtorrent-$(RTORRENT_VERSION)
+RTORRENT_TARBALL := rtorrent-$(RTORRENT_VERSION).tar.gz
+RTORRENT_URL := http://libtorrent.rakshasa.no/downloads/$(RTORRENT_TARBALL)
+#RTORRENT_URL := http://download.systemimager.org/pub/bittorrent/$(RTORRENT_TARBALL)
+RTORRENT_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/rtorrent.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL)
+
+PHONY += rtorrent
+rtorrent: $(RTORRENT_DIR).build
+
+PHONY += rtorrent_install
+rtorrent_install: $(RTORRENT_DIR).install
+$(RTORRENT_DIR).install: $(RTORRENT_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(RTORRENT_DIR)/src/rtorrent $(INITRD_BUILD_DIR)/bin
+
+
+$(RTORRENT_DIR).build: $(RTORRENT_DIR).unpack
+ cd $(RTORRENT_DIR) && ./configure --without-ncursesw --disable-debug
+ cd $(RTORRENT_DIR) && make
+ touch $@
+
+$(RTORRENT_DIR).unpack: $(INITRD_DIR)/make.d/rtorrent.rul \
+ $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL) \
+ $(RTORRENT_PATCHES)
+ rm -rf $(RTORRENT_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(RTORRENT_TARBALL)
+ cd $(RTORRENT_DIR) && cat $(RTORRENT_PATCHES) < /dev/null | patch -p1
+ touch $@
+
+$(INITRD_SRC_DIR)/$(RTORRENT_TARBALL):
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(RTORRENT_URL) $(INITRD_SRC_DIR)
+
+PHONY += rtorrent_clean
+rtorrent_clean:
+ rm -rf $(RTORRENT_DIR)
+ rm -f $(RTORRENT_DIR).unpack
+ rm -f $(RTORRENT_DIR).build
+ rm -f $(RTORRENT_DIR).install
+
Index: initrd_source/make.d/dtach.rul
===================================================================
--- initrd_source/make.d/dtach.rul (revision 0)
+++ initrd_source/make.d/dtach.rul (revision 0)
@@ -0,0 +1,48 @@
+#
+# $Id$
+# vi: set filetype=make:
+#
+
+DTACH_VERSION := 0.7
+DTACH_DIR := $(INITRD_SRC_DIR)/dtach-$(DTACH_VERSION)
+DTACH_TARBALL := dtach-$(DTACH_VERSION).tar.gz
+DTACH_URL := http://switch.dl.sourceforge.net/sourceforge/dtach/$(DTACH_TARBALL)
+#DTACH_URL := http://download.systemimager.org/pub/bittorrent/$(DTACH_TARBALL)
+DTACH_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/dtach.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(DTACH_TARBALL)
+
+PHONY += dtach
+dtach: $(DTACH_DIR).build
+
+PHONY += dtach_install
+dtach_install: $(DTACH_DIR).install
+$(DTACH_DIR).install: $(DTACH_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(DTACH_DIR)/dtach $(INITRD_BUILD_DIR)/bin
+
+
+$(DTACH_DIR).build: $(DTACH_DIR).unpack
+ cd $(DTACH_DIR) && ./configure
+ cd $(DTACH_DIR) && make
+ touch $@
+
+$(DTACH_DIR).unpack: $(INITRD_DIR)/make.d/dtach.rul \
+ $(INITRD_SRC_DIR)/$(DTACH_TARBALL) \
+ $(DTACH_PATCHES)
+ rm -rf $(DTACH_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(DTACH_TARBALL)
+ cd $(DTACH_DIR) && cat $(DTACH_PATCHES) < /dev/null | patch -p1
+ touch $@
+
+$(INITRD_SRC_DIR)/$(DTACH_TARBALL):
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(DTACH_URL) $(INITRD_SRC_DIR)
+
+PHONY += dtach_clean
+dtach_clean:
+ rm -rf $(DTACH_DIR)
+ rm -f $(DTACH_DIR).unpack
+ rm -f $(DTACH_DIR).build
+ rm -f $(DTACH_DIR).install
+
Index: initrd_source/initrd.rul
===================================================================
--- initrd_source/initrd.rul (revision 4062)
+++ initrd_source/initrd.rul (working copy)
@@ -61,6 +61,8 @@
#
include $(INITRD_DIR)/make.d/cx_freeze.rul
include $(INITRD_DIR)/make.d/bittorrent.rul
+include $(INITRD_DIR)/make.d/dtach.rul
+include $(INITRD_DIR)/make.d/rtorrent.rul
include $(INITRD_DIR)/make.d/busybox.rul
include $(INITRD_DIR)/make.d/coreutils.rul
include $(INITRD_DIR)/make.d/dhclient.rul
@@ -126,6 +128,8 @@
$(DHCLIENT_DIR).install \
$(RSYNC_DIR).install \
$(BITTORRENT_DIR).install \
+ $(DTACH_DIR).install \
+ $(RTORRENT_DIR).install \
$(CX_FREEZE_BINARY) \
$(UDPCAST_BINARY)

@@ -139,14 +143,14 @@
ifeq ($(ARCH),x86_64)
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib -v -d lib64 ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib:/usr/local/lib:/opt/gnome/lib -v -d lib64 ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib64/libnss* $(INITRD_BUILD_DIR)/lib64
else
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64 -v -d lib ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64:/usr/local/lib:/opt/gnome/lib -v -d lib ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib/libnss* $(INITRD_BUILD_DIR)/lib
Index: initrd_source/skel/etc/init.d/functions
===================================================================
--- initrd_source/skel/etc/init.d/functions (revision 4062)
+++ initrd_source/skel/etc/init.d/functions (working copy)
@@ -547,11 +547,19 @@
# Minimum upload rate threshold (in KB/s), if lesser stop seeding
bittorrent_upload_min=${BITTORRENT_UPLOAD_MIN:-50}

+ # dtach requires devpts: be sure it's properly mounted.
+ # XXX: honestly, I don't know why an additional remount is needed
+ # here... -AR-
+ mount -t devpts,remount devpts /dev/pts
+
# Start downloading.
- /bin/bittorrent-console --no_upnp --no_start_trackerless_client --max_upload_rate 0 --display_interval 1 --rerequest_interval 1 --bind ${IPADDR} --save_in ${destination} ${torrent} > $bittorrent_log &
+ dtach -n /tmp/socket-`basename ${torrent}` \
+ /bin/rtorrent -d ${destination} \
+ -o tracker_dump=${bittorrent_log},download_rate=0,upload_rate=0 \
+ ${torrent} </dev/console >/dev/console 2>&1 &
pid=$!
if [ ! -d /proc/$pid ]; then
- logmsg "error: couldn't run bittorrent-console!"
+ logmsg "error: couldn't run rtorrent!"
shellout
fi
unset pid
@@ -562,27 +570,9 @@
done

# Checking download...
+ logmsg "download started..."
while :; do
- while :; do
- status=`grep 'percent done:' $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//' | sed -ne '/^[0-9]*\.[0-9]*$/p'`
- [ ! -z "$status" ] && break
- done
- logmsg "percent done: $status %"
- if [ "$status" = "100.0" ]; then
- # Sleep until upload rate reaches the minimum threshold
- while [ "$bittorrent_seed_wait" = "y" ]; do
- sleep $bittorrent_polling_time
- while :; do
- upload_rate=`grep 'upload rate:' $bittorrent_log | sed -ne '$p' | sed 's/upload rate: *\([0-9]*\)\.[0-9]* .*$/\1/' | sed -ne '/^\([0-9]*\)$/p'`
- [ ! -z $upload_rate ] && break
- done
- logmsg "upload rate: $upload_rate KB/s"
- [ $upload_rate -lt $bittorrent_upload_min ] && break
- done
- logmsg "Download completed"
- unset bittorrent_log upload_rate counter
- break
- fi
+ grep "event=completed" ${bittorrent_log} && break
sleep $bittorrent_polling_time
done

@@ -602,10 +592,10 @@
bittorrent_stop() {
# Try to kill all the BitTorrent processes
counter=1
- while ps | grep bittorrent-console | grep -qv grep; do
+ while ps | grep rtorrent | grep -qv grep; do
[ $counter -ge 10 ] && logmsg "error: couldn't kill BitTorrent client!" && shellout
logmsg "killing BitTorrent client..."
- killall -9 bittorrent-console
+ killall -9 dtach rtorrent
sleep 1
counter=$(($counter + 1))
done
Bernard Li
2007-05-17 17:09:04 UTC
Permalink
Hi Andrea:

If your tests show that rtorrent has better performance, let's just
replace the standard python client.

Is rtorrent actively being developed? It would be a shame if it
becomes idle when we are depending on it.

Thanks,

Bernard
Post by Andrea Righi
Hi all,
the following patch includes the rtorrent client
(http://libtorrent.rakshasa.no/) into the initrd.img and uses it instead of the
standard BitTorrent client. This would allow to remove the tricky stuff to
freeze the python scripts, since it's fully implemented in C++. As reported in
the rtorrent website, this client is designed with a particular attention on
high performance and this is just what we really need (the standard BT client
doesn't seem to be able to exploit all the available bandwidth, and in our cases
choking algorithms, fairness, etc. don't seem to be very useful stuff).
Tests with a single imaging client looks very promising! I can reach more than
2x of the download rate, respect to the standard client. ok, well... 1 client is
not a meaningful test for a p2p protocol... :-) I hope I'll be able to test it
in a bigger environment soon...
If you can/want to test it RPMs will be available soon at
http://download.systemimager.org/~arighi/systemimager (look for *_rtorrent*
versions).
Before merging it into the trunk I would like to rewrite it better, in
particular, I'd like to add an option that allows to select rtorrent or standard
BitTorrent clients (like RTORRENT=y|n), to switch from a client to another
without recompiling and easily do some comparisons.
-Andrea
Build Requirements: sigc++-2.0 libtorrent >= 0.11.1
---
Index: make.d/boel_binaries.inc
===================================================================
--- make.d/boel_binaries.inc (revision 4062)
+++ make.d/boel_binaries.inc (working copy)
@@ -137,7 +137,7 @@
TGTLIBDIR=lib ; \
test ! -d /lib64 || TGTLIBDIR=lib64 ; \
cd $(BOEL_BINARIES_DIR) \
- && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*
+ && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/usr/local/lib:/opt/gnome/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*
ifdef IS_PPC64
# Copy all the libnss*. It seems that with ppc64 these are not
Index: initrd_source/make.d/rtorrent.rul
===================================================================
--- initrd_source/make.d/rtorrent.rul (revision 0)
+++ initrd_source/make.d/rtorrent.rul (revision 0)
@@ -0,0 +1,51 @@
+#
+# $Id$
+#
+# 2007-05-14 Andrea Righi
+# - rtorrent binaries
+#
+
+RTORRENT_VERSION := 0.7.1
+RTORRENT_DIR := $(INITRD_SRC_DIR)/rtorrent-$(RTORRENT_VERSION)
+RTORRENT_TARBALL := rtorrent-$(RTORRENT_VERSION).tar.gz
+RTORRENT_URL := http://libtorrent.rakshasa.no/downloads/$(RTORRENT_TARBALL)
+#RTORRENT_URL := http://download.systemimager.org/pub/bittorrent/$(RTORRENT_TARBALL)
+RTORRENT_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/rtorrent.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL)
+
+PHONY += rtorrent
+rtorrent: $(RTORRENT_DIR).build
+
+PHONY += rtorrent_install
+rtorrent_install: $(RTORRENT_DIR).install
+$(RTORRENT_DIR).install: $(RTORRENT_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(RTORRENT_DIR)/src/rtorrent $(INITRD_BUILD_DIR)/bin
+
+
+$(RTORRENT_DIR).build: $(RTORRENT_DIR).unpack
+ cd $(RTORRENT_DIR) && ./configure --without-ncursesw --disable-debug
+ cd $(RTORRENT_DIR) && make
+
+$(RTORRENT_DIR).unpack: $(INITRD_DIR)/make.d/rtorrent.rul \
+ $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL) \
+ $(RTORRENT_PATCHES)
+ rm -rf $(RTORRENT_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(RTORRENT_TARBALL)
+ cd $(RTORRENT_DIR) && cat $(RTORRENT_PATCHES) < /dev/null | patch -p1
+
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(RTORRENT_URL) $(INITRD_SRC_DIR)
+
+PHONY += rtorrent_clean
+ rm -rf $(RTORRENT_DIR)
+ rm -f $(RTORRENT_DIR).unpack
+ rm -f $(RTORRENT_DIR).build
+ rm -f $(RTORRENT_DIR).install
+
Index: initrd_source/make.d/dtach.rul
===================================================================
--- initrd_source/make.d/dtach.rul (revision 0)
+++ initrd_source/make.d/dtach.rul (revision 0)
@@ -0,0 +1,48 @@
+#
+# $Id$
+#
+
+DTACH_VERSION := 0.7
+DTACH_DIR := $(INITRD_SRC_DIR)/dtach-$(DTACH_VERSION)
+DTACH_TARBALL := dtach-$(DTACH_VERSION).tar.gz
+DTACH_URL := http://switch.dl.sourceforge.net/sourceforge/dtach/$(DTACH_TARBALL)
+#DTACH_URL := http://download.systemimager.org/pub/bittorrent/$(DTACH_TARBALL)
+DTACH_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/dtach.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(DTACH_TARBALL)
+
+PHONY += dtach
+dtach: $(DTACH_DIR).build
+
+PHONY += dtach_install
+dtach_install: $(DTACH_DIR).install
+$(DTACH_DIR).install: $(DTACH_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(DTACH_DIR)/dtach $(INITRD_BUILD_DIR)/bin
+
+
+$(DTACH_DIR).build: $(DTACH_DIR).unpack
+ cd $(DTACH_DIR) && ./configure
+ cd $(DTACH_DIR) && make
+
+$(DTACH_DIR).unpack: $(INITRD_DIR)/make.d/dtach.rul \
+ $(INITRD_SRC_DIR)/$(DTACH_TARBALL) \
+ $(DTACH_PATCHES)
+ rm -rf $(DTACH_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(DTACH_TARBALL)
+ cd $(DTACH_DIR) && cat $(DTACH_PATCHES) < /dev/null | patch -p1
+
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(DTACH_URL) $(INITRD_SRC_DIR)
+
+PHONY += dtach_clean
+ rm -rf $(DTACH_DIR)
+ rm -f $(DTACH_DIR).unpack
+ rm -f $(DTACH_DIR).build
+ rm -f $(DTACH_DIR).install
+
Index: initrd_source/initrd.rul
===================================================================
--- initrd_source/initrd.rul (revision 4062)
+++ initrd_source/initrd.rul (working copy)
@@ -61,6 +61,8 @@
#
include $(INITRD_DIR)/make.d/cx_freeze.rul
include $(INITRD_DIR)/make.d/bittorrent.rul
+include $(INITRD_DIR)/make.d/dtach.rul
+include $(INITRD_DIR)/make.d/rtorrent.rul
include $(INITRD_DIR)/make.d/busybox.rul
include $(INITRD_DIR)/make.d/coreutils.rul
include $(INITRD_DIR)/make.d/dhclient.rul
@@ -126,6 +128,8 @@
$(DHCLIENT_DIR).install \
$(RSYNC_DIR).install \
$(BITTORRENT_DIR).install \
+ $(DTACH_DIR).install \
+ $(RTORRENT_DIR).install \
$(CX_FREEZE_BINARY) \
$(UDPCAST_BINARY)
@@ -139,14 +143,14 @@
ifeq ($(ARCH),x86_64)
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib -v -d lib64 ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib:/usr/local/lib:/opt/gnome/lib -v -d lib64 ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib64/libnss* $(INITRD_BUILD_DIR)/lib64
else
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64 -v -d lib ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64:/usr/local/lib:/opt/gnome/lib -v -d lib ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib/libnss* $(INITRD_BUILD_DIR)/lib
Index: initrd_source/skel/etc/init.d/functions
===================================================================
--- initrd_source/skel/etc/init.d/functions (revision 4062)
+++ initrd_source/skel/etc/init.d/functions (working copy)
@@ -547,11 +547,19 @@
# Minimum upload rate threshold (in KB/s), if lesser stop seeding
bittorrent_upload_min=${BITTORRENT_UPLOAD_MIN:-50}
+ # dtach requires devpts: be sure it's properly mounted.
+ # XXX: honestly, I don't know why an additional remount is needed
+ # here... -AR-
+ mount -t devpts,remount devpts /dev/pts
+
# Start downloading.
- /bin/bittorrent-console --no_upnp --no_start_trackerless_client --max_upload_rate 0 --display_interval 1 --rerequest_interval 1 --bind ${IPADDR} --save_in ${destination} ${torrent} > $bittorrent_log &
+ dtach -n /tmp/socket-`basename ${torrent}` \
+ /bin/rtorrent -d ${destination} \
+ -o tracker_dump=${bittorrent_log},download_rate=0,upload_rate=0 \
+ ${torrent} </dev/console >/dev/console 2>&1 &
pid=$!
if [ ! -d /proc/$pid ]; then
- logmsg "error: couldn't run bittorrent-console!"
+ logmsg "error: couldn't run rtorrent!"
shellout
fi
unset pid
@@ -562,27 +570,9 @@
done
# Checking download...
+ logmsg "download started..."
while :; do
- while :; do
- status=`grep 'percent done:' $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//' | sed -ne '/^[0-9]*\.[0-9]*$/p'`
- [ ! -z "$status" ] && break
- done
- logmsg "percent done: $status %"
- if [ "$status" = "100.0" ]; then
- # Sleep until upload rate reaches the minimum threshold
- while [ "$bittorrent_seed_wait" = "y" ]; do
- sleep $bittorrent_polling_time
- while :; do
- upload_rate=`grep 'upload rate:' $bittorrent_log | sed -ne '$p' | sed 's/upload rate: *\([0-9]*\)\.[0-9]* .*$/\1/' | sed -ne '/^\([0-9]*\)$/p'`
- [ ! -z $upload_rate ] && break
- done
- logmsg "upload rate: $upload_rate KB/s"
- [ $upload_rate -lt $bittorrent_upload_min ] && break
- done
- logmsg "Download completed"
- unset bittorrent_log upload_rate counter
- break
- fi
+ grep "event=completed" ${bittorrent_log} && break
sleep $bittorrent_polling_time
done
@@ -602,10 +592,10 @@
bittorrent_stop() {
# Try to kill all the BitTorrent processes
counter=1
- while ps | grep bittorrent-console | grep -qv grep; do
+ while ps | grep rtorrent | grep -qv grep; do
[ $counter -ge 10 ] && logmsg "error: couldn't kill BitTorrent client!" && shellout
logmsg "killing BitTorrent client..."
- killall -9 bittorrent-console
+ killall -9 dtach rtorrent
sleep 1
counter=$(($counter + 1))
done
Brian Elliott Finley
2007-05-17 20:20:34 UTC
Permalink
Very cool!
Post by Andrea Righi
Hi all,
the following patch includes the rtorrent client
(http://libtorrent.rakshasa.no/) into the initrd.img and uses it instead of the
standard BitTorrent client. This would allow to remove the tricky stuff to
freeze the python scripts, since it's fully implemented in C++. As reported in
the rtorrent website, this client is designed with a particular attention on
high performance and this is just what we really need (the standard BT client
doesn't seem to be able to exploit all the available bandwidth, and in our cases
choking algorithms, fairness, etc. don't seem to be very useful stuff).
Tests with a single imaging client looks very promising! I can reach more than
2x of the download rate, respect to the standard client. ok, well... 1 client is
not a meaningful test for a p2p protocol... :-) I hope I'll be able to test it
in a bigger environment soon...
If you can/want to test it RPMs will be available soon at
http://download.systemimager.org/~arighi/systemimager (look for *_rtorrent*
versions).
Before merging it into the trunk I would like to rewrite it better, in
particular, I'd like to add an option that allows to select rtorrent or standard
BitTorrent clients (like RTORRENT=y|n), to switch from a client to another
without recompiling and easily do some comparisons.
-Andrea
Build Requirements: sigc++-2.0 libtorrent >= 0.11.1
---
Index: make.d/boel_binaries.inc
===================================================================
--- make.d/boel_binaries.inc (revision 4062)
+++ make.d/boel_binaries.inc (working copy)
@@ -137,7 +137,7 @@
TGTLIBDIR=lib ; \
test ! -d /lib64 || TGTLIBDIR=lib64 ; \
cd $(BOEL_BINARIES_DIR) \
- && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*
+ && $(PYTHON) $(TOPDIR)/initrd_source/mklibs -L $(SRC_DIR)/$(PARTED_DIR)/libparted/.libs:$(DISCOVER_DIR)/lib/.libs:$(SRC_DIR)/$(DEVMAPPER_DIR)/lib/ioctl:$(SRC_DIR)/$(E2FSPROGS_DIR)/lib:/lib64:/usr/lib64:/usr/kerberos/lib64:/lib:/usr/lib:/usr/kerberos/lib:/usr/local/lib:/opt/gnome/lib:/lib/tls:`find /lib/tls /usr/lib/tls /lib64/tls /usr/lib64/tls -type d 2>/dev/null | tr '\n' :` -v -d $$TGTLIBDIR bin/* sbin/* usr/bin/* usr/sbin/* $(INITRD_BUILD_DIR)/*bin/* $(INITRD_BUILD_DIR)/usr/*bin/*
ifdef IS_PPC64
# Copy all the libnss*. It seems that with ppc64 these are not
Index: initrd_source/make.d/rtorrent.rul
===================================================================
--- initrd_source/make.d/rtorrent.rul (revision 0)
+++ initrd_source/make.d/rtorrent.rul (revision 0)
@@ -0,0 +1,51 @@
+#
+# $Id$
+#
+# 2007-05-14 Andrea Righi
+# - rtorrent binaries
+#
+
+RTORRENT_VERSION := 0.7.1
+RTORRENT_DIR := $(INITRD_SRC_DIR)/rtorrent-$(RTORRENT_VERSION)
+RTORRENT_TARBALL := rtorrent-$(RTORRENT_VERSION).tar.gz
+RTORRENT_URL := http://libtorrent.rakshasa.no/downloads/$(RTORRENT_TARBALL)
+#RTORRENT_URL := http://download.systemimager.org/pub/bittorrent/$(RTORRENT_TARBALL)
+RTORRENT_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/rtorrent.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL)
+
+PHONY += rtorrent
+rtorrent: $(RTORRENT_DIR).build
+
+PHONY += rtorrent_install
+rtorrent_install: $(RTORRENT_DIR).install
+$(RTORRENT_DIR).install: $(RTORRENT_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(RTORRENT_DIR)/src/rtorrent $(INITRD_BUILD_DIR)/bin
+
+
+$(RTORRENT_DIR).build: $(RTORRENT_DIR).unpack
+ cd $(RTORRENT_DIR) && ./configure --without-ncursesw --disable-debug
+ cd $(RTORRENT_DIR) && make
+
+$(RTORRENT_DIR).unpack: $(INITRD_DIR)/make.d/rtorrent.rul \
+ $(INITRD_SRC_DIR)/$(RTORRENT_TARBALL) \
+ $(RTORRENT_PATCHES)
+ rm -rf $(RTORRENT_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(RTORRENT_TARBALL)
+ cd $(RTORRENT_DIR) && cat $(RTORRENT_PATCHES) < /dev/null | patch -p1
+
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(RTORRENT_URL) $(INITRD_SRC_DIR)
+
+PHONY += rtorrent_clean
+ rm -rf $(RTORRENT_DIR)
+ rm -f $(RTORRENT_DIR).unpack
+ rm -f $(RTORRENT_DIR).build
+ rm -f $(RTORRENT_DIR).install
+
Index: initrd_source/make.d/dtach.rul
===================================================================
--- initrd_source/make.d/dtach.rul (revision 0)
+++ initrd_source/make.d/dtach.rul (revision 0)
@@ -0,0 +1,48 @@
+#
+# $Id$
+#
+
+DTACH_VERSION := 0.7
+DTACH_DIR := $(INITRD_SRC_DIR)/dtach-$(DTACH_VERSION)
+DTACH_TARBALL := dtach-$(DTACH_VERSION).tar.gz
+DTACH_URL := http://switch.dl.sourceforge.net/sourceforge/dtach/$(DTACH_TARBALL)
+#DTACH_URL := http://download.systemimager.org/pub/bittorrent/$(DTACH_TARBALL)
+DTACH_PATCHES := $(shell ls $(INITRD_PATCH_DIR)/dtach.*.patch 2>/dev/null | sort)
+
+ALL_SOURCE += $(INITRD_SRC_DIR)/$(DTACH_TARBALL)
+
+PHONY += dtach
+dtach: $(DTACH_DIR).build
+
+PHONY += dtach_install
+dtach_install: $(DTACH_DIR).install
+$(DTACH_DIR).install: $(DTACH_DIR).build \
+ $(INITRD_BUILD_DIR).prep
+ install -m 755 $(DTACH_DIR)/dtach $(INITRD_BUILD_DIR)/bin
+
+
+$(DTACH_DIR).build: $(DTACH_DIR).unpack
+ cd $(DTACH_DIR) && ./configure
+ cd $(DTACH_DIR) && make
+
+$(DTACH_DIR).unpack: $(INITRD_DIR)/make.d/dtach.rul \
+ $(INITRD_SRC_DIR)/$(DTACH_TARBALL) \
+ $(DTACH_PATCHES)
+ rm -rf $(DTACH_DIR)
+ cd $(INITRD_SRC_DIR) && tar -xvzf $(DTACH_TARBALL)
+ cd $(DTACH_DIR) && cat $(DTACH_PATCHES) < /dev/null | patch -p1
+
+ mkdir -p $(INITRD_SRC_DIR)
+ $(GETSOURCE) $(DTACH_URL) $(INITRD_SRC_DIR)
+
+PHONY += dtach_clean
+ rm -rf $(DTACH_DIR)
+ rm -f $(DTACH_DIR).unpack
+ rm -f $(DTACH_DIR).build
+ rm -f $(DTACH_DIR).install
+
Index: initrd_source/initrd.rul
===================================================================
--- initrd_source/initrd.rul (revision 4062)
+++ initrd_source/initrd.rul (working copy)
@@ -61,6 +61,8 @@
#
include $(INITRD_DIR)/make.d/cx_freeze.rul
include $(INITRD_DIR)/make.d/bittorrent.rul
+include $(INITRD_DIR)/make.d/dtach.rul
+include $(INITRD_DIR)/make.d/rtorrent.rul
include $(INITRD_DIR)/make.d/busybox.rul
include $(INITRD_DIR)/make.d/coreutils.rul
include $(INITRD_DIR)/make.d/dhclient.rul
@@ -126,6 +128,8 @@
$(DHCLIENT_DIR).install \
$(RSYNC_DIR).install \
$(BITTORRENT_DIR).install \
+ $(DTACH_DIR).install \
+ $(RTORRENT_DIR).install \
$(CX_FREEZE_BINARY) \
$(UDPCAST_BINARY)
@@ -139,14 +143,14 @@
ifeq ($(ARCH),x86_64)
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib -v -d lib64 ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib64:/usr/lib64:/lib:/usr/lib:/usr/local/lib:/opt/gnome/lib -v -d lib64 ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib64/libnss* $(INITRD_BUILD_DIR)/lib64
else
( \
cd $(INITRD_BUILD_DIR) && \
- $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64 -v -d lib ./*bin/* \
+ $(PYTHON) $(INITRD_DIR)/mklibs -L /lib:/usr/lib:/lib64:/usr/lib64:/usr/local/lib:/opt/gnome/lib -v -d lib ./*bin/* \
)
# copy libnss files as they are not automatically included
cp -a /lib/libnss* $(INITRD_BUILD_DIR)/lib
Index: initrd_source/skel/etc/init.d/functions
===================================================================
--- initrd_source/skel/etc/init.d/functions (revision 4062)
+++ initrd_source/skel/etc/init.d/functions (working copy)
@@ -547,11 +547,19 @@
# Minimum upload rate threshold (in KB/s), if lesser stop seeding
bittorrent_upload_min=${BITTORRENT_UPLOAD_MIN:-50}
+ # dtach requires devpts: be sure it's properly mounted.
+ # XXX: honestly, I don't know why an additional remount is needed
+ # here... -AR-
+ mount -t devpts,remount devpts /dev/pts
+
# Start downloading.
- /bin/bittorrent-console --no_upnp --no_start_trackerless_client --max_upload_rate 0 --display_interval 1 --rerequest_interval 1 --bind ${IPADDR} --save_in ${destination} ${torrent} > $bittorrent_log &
+ dtach -n /tmp/socket-`basename ${torrent}` \
+ /bin/rtorrent -d ${destination} \
+ -o tracker_dump=${bittorrent_log},download_rate=0,upload_rate=0 \
+ ${torrent} </dev/console >/dev/console 2>&1 &
pid=$!
if [ ! -d /proc/$pid ]; then
- logmsg "error: couldn't run bittorrent-console!"
+ logmsg "error: couldn't run rtorrent!"
shellout
fi
unset pid
@@ -562,27 +570,9 @@
done
# Checking download...
+ logmsg "download started..."
while :; do
- while :; do
- status=`grep 'percent done:' $bittorrent_log | sed -ne '$p' | sed 's/percent done: *//' | sed -ne '/^[0-9]*\.[0-9]*$/p'`
- [ ! -z "$status" ] && break
- done
- logmsg "percent done: $status %"
- if [ "$status" = "100.0" ]; then
- # Sleep until upload rate reaches the minimum threshold
- while [ "$bittorrent_seed_wait" = "y" ]; do
- sleep $bittorrent_polling_time
- while :; do
- upload_rate=`grep 'upload rate:' $bittorrent_log | sed -ne '$p' | sed 's/upload rate: *\([0-9]*\)\.[0-9]* .*$/\1/' | sed -ne '/^\([0-9]*\)$/p'`
- [ ! -z $upload_rate ] && break
- done
- logmsg "upload rate: $upload_rate KB/s"
- [ $upload_rate -lt $bittorrent_upload_min ] && break
- done
- logmsg "Download completed"
- unset bittorrent_log upload_rate counter
- break
- fi
+ grep "event=completed" ${bittorrent_log} && break
sleep $bittorrent_polling_time
done
@@ -602,10 +592,10 @@
bittorrent_stop() {
# Try to kill all the BitTorrent processes
counter=1
- while ps | grep bittorrent-console | grep -qv grep; do
+ while ps | grep rtorrent | grep -qv grep; do
[ $counter -ge 10 ] && logmsg "error: couldn't kill BitTorrent client!" && shellout
logmsg "killing BitTorrent client..."
- killall -9 bittorrent-console
+ killall -9 dtach rtorrent
sleep 1
counter=$(($counter + 1))
done
--
Brian Elliott Finley
Linux Strategist, CIS
Desk: 630.252.4742
Cell: 630.631.6621
Loading...