axel/trunk/axel.c

  /********************************************************************\
  * Axel -- A lighter download accelerator for Linux and other Unices. *
  *                                                                    *
  * Copyright 2001 Wilmer van der Gaast                                *
  \********************************************************************/

/* Main control                                                         */

/*
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License with
  the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
  if not, write to the Free Software Foundation, Inc., 59 Temple Place,
  Suite 330, Boston, MA  02111-1307  USA
*/

#include "axel.h"

/* Axel */
static void save_state( axel_t *axel );
static void *setup_thread( void * );
static void axel_message( axel_t *axel, char *format, ... );
static void axel_divide( axel_t *axel );

static char *buffer = NULL;

/* Create a new axel_t structure                                        */
axel_t *axel_new( conf_t *conf, int count, void *url )
{
        search_t *res;
        axel_t *axel;
        url_t *u;
        char *s;
        int i;
        
        axel = malloc( sizeof( axel_t ) );
        memset( axel, 0, sizeof( axel_t ) );
        *axel->conf = *conf;
        axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
        memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
                {
                        if( axel->conf->verbose >= 2 )
                                axel_message( axel, _("Buffer resized for this speed.") );
                        axel->conf->buffer_size = axel->conf->max_speed;
                }
                axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
        }
        if( buffer == NULL )
                buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
        
        if( count == 0 )
        {
                axel->url = malloc( sizeof( url_t ) );
                axel->url->next = axel->url;
                strncpy( axel->url->text, (char *) url, MAX_STRING );
        }
        else
        {
                res = (search_t *) url;
                u = axel->url = malloc( sizeof( url_t ) );
                for( i = 0; i < count; i ++ )
                {
                        strncpy( u->text, res[i].url, MAX_STRING );
                        if( i < count - 1 )
                        {
                                u->next = malloc( sizeof( url_t ) );
                                u = u->next;
                        }
                        else
                        {
                                u->next = axel->url;
                        }
                }
        }
        
        axel->conn[0].conf = axel->conf;
        if( !conn_set( &axel->conn[0], axel->url->text ) )
        {
                axel_message( axel, _("Could not parse URL.\n") );
                axel->ready = -1;
                return( axel );
        }

        axel->conn[0].local_if = axel->conf->interfaces->text;
        axel->conf->interfaces = axel->conf->interfaces->next;
        
        strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        http_decode( axel->filename );
        if( *axel->filename == 0 )      /* Index page == no fn          */
                strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
        if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
                *s = 0;         /* Get rid of CGI parameters            */
        
        if( !conn_init( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        
        /* This does more than just checking the file size, it all depends
           on the protocol used.                                        */
        if( !conn_info( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        s = conn_url( axel->conn );
        strncpy( axel->url->text, s, MAX_STRING );
        if( ( axel->size = axel->conn[0].size ) != INT_MAX )
        {
                if( axel->conf->verbose > 0 )
                        axel_message( axel, _("File size: %lld bytes"), axel->size );
        }
        
        /* Wildcards in URL --> Get complete filename                   */
        if( strchr( axel->filename, '*' ) || strchr( axel->filename, '?' ) )
                strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        
        return( axel );
}

/* Open a local file to store the downloaded data                       */
int axel_open( axel_t *axel )
{
        int i, fd;
        long long int j;
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Opening output file %s"), axel->filename );
        snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
        
        axel->outfd = -1;
        
        /* Check whether server knows about RESTart and switch back to
           single connection download if necessary                      */
        if( !axel->conn[0].supported )
        {
                axel_message( axel, _("Server unsupported, "
                        "starting from scratch with one connection.") );
                axel->conf->num_connections = 1;
                axel->conn = realloc( axel->conn, sizeof( conn_t ) );
                axel_divide( axel );
        }
        else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
        {
                read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
                
                axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
                memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );

                axel_divide( axel );
                
                read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
                for( i = 0; i < axel->conf->num_connections; i ++ )
                        read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );

                axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
                        axel->bytes_done, axel->size - axel->bytes_done );
                
                close( fd );
                
                if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
        }

        /* If outfd == -1 we have to start from scrath now              */
        if( axel->outfd == -1 )
        {
                axel_divide( axel );
                
                if( ( axel->outfd = open( axel->filename, O_CREAT | O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
                
                /* And check whether the filesystem can handle seeks to
                   past-EOF areas.. Speeds things up. :) AFAIK this
                   should just not happen:                              */
                if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
                {
                        /* But if the OS/fs does not allow to seek behind
                           EOF, we have to fill the file with zeroes before
                           starting. Slow..                             */
                        axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
                        lseek( axel->outfd, 0, SEEK_SET );
                        memset( buffer, 0, axel->conf->buffer_size );
                        j = axel->size;
                        while( j > 0 )
                        {
                                write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
                                j -= axel->conf->buffer_size;
                        }
                }
        }
        
        return( 1 );
}

/* Start downloading                                                    */
void axel_start( axel_t *axel )
{
        int i;
        
        /* HTTP might've redirected and FTP handles wildcards, so
           re-scan the URL for every conn                               */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                conn_set( &axel->conn[i], axel->url->text );
                axel->url = axel->url->next;
                axel->conn[i].local_if = axel->conf->interfaces->text;
                axel->conf->interfaces = axel->conf->interfaces->next;
                axel->conn[i].conf = axel->conf;
                if( i ) axel->conn[i].supported = 1;
        }
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Starting download") );
        
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
        {
                if( axel->conf->verbose >= 2 )
                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
                {
                        axel_message( axel, _("pthread error!!!") );
                        axel->ready = -1;
                }
                else
                {
                        axel->conn[i].last_transfer = gettime();
                        axel->conn[i].state = 1;
                }
        }
        
        /* The real downloading will start now, so let's start counting */
        axel->start_time = gettime();
        axel->ready = 0;
}

/* Main 'loop'                                                          */
void axel_do( axel_t *axel )
{
        fd_set fds[1];
        int hifd, i;
        long long int remaining,size;
        struct timeval timeval[1];
        
        /* Create statefile if necessary                                */
        if( gettime() > axel->next_state )
        {
                save_state( axel );
                axel->next_state = gettime() + axel->conf->save_state_interval;
        }
        
        /* Wait for data on (one of) the connections                    */
        FD_ZERO( fds );
        hifd = 0;
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( axel->conn[i].enabled )
                        FD_SET( axel->conn[i].fd, fds );
                hifd = max( hifd, axel->conn[i].fd );
        }
        if( hifd == 0 )
        {
                /* No connections yet. Wait...                          */
                usleep( 100000 );
                goto conn_check;
        }
        else
        {
                timeval->tv_sec = 0;
                timeval->tv_usec = 100000;
                /* A select() error probably means it was interrupted
                   by a signal, or that something else's very wrong...  */
                if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
                {
                        axel->ready = -1;
                        return;
                }
        }
        
        /* Handle connections which need attention                      */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].enabled ) {
        if( FD_ISSET( axel->conn[i].fd, fds ) )
        {
                axel->conn[i].last_transfer = gettime();
                size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
                if( size == -1 )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Error on connection %i! "
                                        "Connection closed"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                else if( size == 0 )
                {
                        if( axel->conf->verbose )
                        {
                                /* Only abnormal behaviour if:          */
                                if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
                                {
                                        axel_message( axel, _("Connection %i unexpectedly closed"), i );
                                }
                                else
                                {
                                        axel_message( axel, _("Connection %i finished"), i );
                                }
                        }
                        if( !axel->conn[0].supported )
                        {
                                axel->ready = 1;
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                /* remaining == Bytes to go                                     */
                remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
                if( remaining < size )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Connection %i finished"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        size = remaining;
                        /* Don't terminate, still stuff to write!       */
                }
                /* This should always succeed..                         */
                lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
                if( write( axel->outfd, buffer, size ) != size )
                {
                        
                        axel_message( axel, _("Write error!") );
                        axel->ready = -1;
                        return;
                }
                axel->conn[i].currentbyte += size;
                axel->bytes_done += size;
        }
        else
        {
                if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
                {
                        if( axel->conf->verbose )
                                axel_message( axel, _("Connection %i timed out"), i );
                        conn_disconnect( &axel->conn[i] );
                        axel->conn[i].enabled = 0;
                }
        } }
        
        if( axel->ready )
                return;
        
conn_check:
        /* Look for aborted connections and attempt to restart them.    */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
                {
                        if( axel->conn[i].state == 0 )
                        {       
                                // Wait for termination of this thread
                                pthread_join(axel->conn[i].setup_thread, NULL);
                                
                                conn_set( &axel->conn[i], axel->url->text );
                                axel->url = axel->url->next;
                                /* axel->conn[i].local_if = axel->conf->interfaces->text;
                                axel->conf->interfaces = axel->conf->interfaces->next; */
                                if( axel->conf->verbose >= 2 )
                                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
                                {
                                        axel->conn[i].state = 1;
                                        axel->conn[i].last_transfer = gettime();
                                }
                                else
                                {
                                        axel_message( axel, _("pthread error!!!") );
                                        axel->ready = -1;
                                }
                        }
                        else
                        {
                                if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
                                {
                                        pthread_cancel( *axel->conn[i].setup_thread );
                                        axel->conn[i].state = 0;
                                }
                        }
                }
        }

        /* Calculate current average speed and finish_time              */
        axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
        axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );

        /* Check speed. If too high, delay for some time to slow things
           down a bit. I think a 5% deviation should be acceptable.     */
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
                        axel->delay_time += 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
                        axel->delay_time -= 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
                        axel->delay_time = 0;
                usleep( axel->delay_time );
        }
        
        /* Ready?                                                       */
        if( axel->bytes_done == axel->size )
                axel->ready = 1;
}

/* Close an axel connection                                             */
void axel_close( axel_t *axel )
{
        int i;
        message_t *m;
        
        /* Terminate any thread still running                           */
        for( i = 0; i < axel->conf->num_connections; i ++ )
                /* don't try to kill non existing thread */
                if ( *axel->conn[i].setup_thread != 0 )
                        pthread_cancel( *axel->conn[i].setup_thread );
        
        /* Delete state file if necessary                               */
        if( axel->ready == 1 )
        {
                snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
                unlink( buffer );
        }
        /* Else: Create it..                                            */
        else if( axel->bytes_done > 0 )
        {
                save_state( axel );
        }

        /* Delete any message not processed yet                         */
        while( axel->message )
        {
                m = axel->message;
                axel->message = axel->message->next;
                free( m );
        }
        
        /* Close all connections and local file                         */
        close( axel->outfd );
        for( i = 0; i < axel->conf->num_connections; i ++ )
                conn_disconnect( &axel->conn[i] );

        free( axel->conn );
        free( axel );
}

/* time() with more precision                                           */
double gettime()
{
        struct timeval time[1];
        
        gettimeofday( time, 0 );
        return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
}

/* Save the state of the current download                               */
void save_state( axel_t *axel )
{
        int fd, i;
        char fn[MAX_STRING+4];

        /* No use for such a file if the server doesn't support
           resuming anyway..                                            */
        if( !axel->conn[0].supported )
                return;
        
        snprintf( fn, MAX_STRING, "%s.st", axel->filename );
        if( ( fd = open( fn, O_CREAT | O_TRUNC | O_WRONLY, 0666 ) ) == -1 )
        {
                return;         /* Not 100% fatal..                     */
        }
        write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
        write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
        }
        close( fd );
}

/* Thread used to set up a connection                                   */
void *setup_thread( void *c )
{
        conn_t *conn = c;
        int oldstate;
        
        /* Allow this thread to be killed at any time.                  */
        pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
        pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
        
        if( conn_setup( conn ) )
        {
                conn->last_transfer = gettime();
                if( conn_exec( conn ) )
                {
                        conn->last_transfer = gettime();
                        conn->enabled = 1;
                        conn->state = 0;
                        return( NULL );
                }
        }
        
        conn_disconnect( conn );
        conn->state = 0;
        return( NULL );
}

/* Add a message to the axel->message structure                         */
static void axel_message( axel_t *axel, char *format, ... )
{
        message_t *m = malloc( sizeof( message_t ) ), *n = axel->message;
        va_list params;
        
        memset( m, 0, sizeof( message_t ) );
        va_start( params, format );
        vsnprintf( m->text, MAX_STRING, format, params );
        va_end( params );
        
        if( axel->message == NULL )
        {
                axel->message = m;
        }
        else
        {
                while( n->next != NULL )
                        n = n->next;
                n->next = m;
        }
}

/* Divide the file and set the locations for each connection            */
static void axel_divide( axel_t *axel )
{
        int i;
        
        axel->conn[0].currentbyte = 0;
        axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
        for( i = 1; i < axel->conf->num_connections; i ++ )
        {
#ifdef DEBUG
                printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
                axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
                axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
        }
        axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
#ifdef DEBUG
        printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
}
1	appaji-guest	2	/********************************************************************\
2			* Axel -- A lighter download accelerator for Linux and other Unices. *
3			* *
4			* Copyright 2001 Wilmer van der Gaast *
5			\********************************************************************/
6
7			/* Main control */
8
9			/*
10			This program is free software; you can redistribute it and/or modify
11			it under the terms of the GNU General Public License as published by
12			the Free Software Foundation; either version 2 of the License, or
13			(at your option) any later version.
14
15			This program is distributed in the hope that it will be useful,
16			but WITHOUT ANY WARRANTY; without even the implied warranty of
17			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			GNU General Public License for more details.
19
20			You should have received a copy of the GNU General Public License with
21			the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
22			if not, write to the Free Software Foundation, Inc., 59 Temple Place,
23			Suite 330, Boston, MA 02111-1307 USA
24			*/
25
26			#include "axel.h"
27
28			/* Axel */
29			static void save_state( axel_t *axel );
30			static void setup_thread( void );
31			static void axel_message( axel_t axel, char format, ... );
32			static void axel_divide( axel_t *axel );
33
34			static char *buffer = NULL;
35
36			/* Create a new axel_t structure */
37			axel_t axel_new( conf_t conf, int count, void *url )
38			{
39			search_t *res;
40			axel_t *axel;
41			url_t *u;
42			char *s;
43			int i;
44
45			axel = malloc( sizeof( axel_t ) );
46			memset( axel, 0, sizeof( axel_t ) );
47			axel->conf = conf;
48			axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
49			memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
50			if( axel->conf->max_speed > 0 )
51			{
52			if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
53			{
54			if( axel->conf->verbose >= 2 )
55			axel_message( axel, _("Buffer resized for this speed.") );
56			axel->conf->buffer_size = axel->conf->max_speed;
57			}
58			axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
59			}
60			if( buffer == NULL )
61			buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
62
63			if( count == 0 )
64			{
65			axel->url = malloc( sizeof( url_t ) );
66			axel->url->next = axel->url;
67	appaji-guest	14	strncpy( axel->url->text, (char *) url, MAX_STRING );
68	appaji-guest	2	}
69			else
70			{
71			res = (search_t *) url;
72			u = axel->url = malloc( sizeof( url_t ) );
73			for( i = 0; i < count; i ++ )
74			{
75	appaji-guest	14	strncpy( u->text, res[i].url, MAX_STRING );
76	appaji-guest	2	if( i < count - 1 )
77			{
78			u->next = malloc( sizeof( url_t ) );
79			u = u->next;
80			}
81			else
82			{
83			u->next = axel->url;
84			}
85			}
86			}
87
88			axel->conn[0].conf = axel->conf;
89			if( !conn_set( &axel->conn[0], axel->url->text ) )
90			{
91			axel_message( axel, _("Could not parse URL.\n") );
92			axel->ready = -1;
93			return( axel );
94			}
95
96			axel->conn[0].local_if = axel->conf->interfaces->text;
97			axel->conf->interfaces = axel->conf->interfaces->next;
98
99	appaji-guest	14	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
100	appaji-guest	2	http_decode( axel->filename );
101			if( axel->filename == 0 ) / Index page == no fn */
102	appaji-guest	14	strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
103	appaji-guest	2	if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
104			s = 0; / Get rid of CGI parameters */
105
106			if( !conn_init( &axel->conn[0] ) )
107			{
108			axel_message( axel, axel->conn[0].message );
109			axel->ready = -1;
110			return( axel );
111			}
112
113			/* This does more than just checking the file size, it all depends
114			on the protocol used. */
115			if( !conn_info( &axel->conn[0] ) )
116			{
117			axel_message( axel, axel->conn[0].message );
118			axel->ready = -1;
119			return( axel );
120			}
121			s = conn_url( axel->conn );
122	appaji-guest	14	strncpy( axel->url->text, s, MAX_STRING );
123	appaji-guest	2	if( ( axel->size = axel->conn[0].size ) != INT_MAX )
124			{
125			if( axel->conf->verbose > 0 )
126	appaji-guest	25	axel_message( axel, _("File size: %lld bytes"), axel->size );
127	appaji-guest	2	}
128
129			/* Wildcards in URL --> Get complete filename */
130			if( strchr( axel->filename, '*' ) \|\| strchr( axel->filename, '?' ) )
131	appaji-guest	14	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
132	appaji-guest	2
133			return( axel );
134			}
135
136			/* Open a local file to store the downloaded data */
137			int axel_open( axel_t *axel )
138			{
139			int i, fd;
140	appaji-guest	25	long long int j;
141	appaji-guest	2
142			if( axel->conf->verbose > 0 )
143			axel_message( axel, _("Opening output file %s"), axel->filename );
144			snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
145
146			axel->outfd = -1;
147
148			/* Check whether server knows about RESTart and switch back to
149			single connection download if necessary */
150			if( !axel->conn[0].supported )
151			{
152			axel_message( axel, _("Server unsupported, "
153			"starting from scratch with one connection.") );
154			axel->conf->num_connections = 1;
155			axel->conn = realloc( axel->conn, sizeof( conn_t ) );
156			axel_divide( axel );
157			}
158			else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
159			{
160			read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
161
162			axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
163			memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );
164
165			axel_divide( axel );
166
167			read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
168			for( i = 0; i < axel->conf->num_connections; i ++ )
169			read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
170
171	appaji-guest	25	axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
172	appaji-guest	2	axel->bytes_done, axel->size - axel->bytes_done );
173
174			close( fd );
175
176			if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
177			{
178			axel_message( axel, _("Error opening local file") );
179			return( 0 );
180			}
181			}
182
183			/* If outfd == -1 we have to start from scrath now */
184			if( axel->outfd == -1 )
185			{
186			axel_divide( axel );
187
188			if( ( axel->outfd = open( axel->filename, O_CREAT \| O_WRONLY, 0666 ) ) == -1 )
189			{
190			axel_message( axel, _("Error opening local file") );
191			return( 0 );
192			}
193
194			/* And check whether the filesystem can handle seeks to
195			past-EOF areas.. Speeds things up. :) AFAIK this
196			should just not happen: */
197	appaji-guest	26	if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
198	appaji-guest	2	{
199			/* But if the OS/fs does not allow to seek behind
200			EOF, we have to fill the file with zeroes before
201			starting. Slow.. */
202			axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
203			lseek( axel->outfd, 0, SEEK_SET );
204			memset( buffer, 0, axel->conf->buffer_size );
205	appaji-guest	25	j = axel->size;
206			while( j > 0 )
207	appaji-guest	2	{
208	appaji-guest	25	write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
209			j -= axel->conf->buffer_size;
210	appaji-guest	2	}
211			}
212			}
213
214			return( 1 );
215			}
216
217			/* Start downloading */
218			void axel_start( axel_t *axel )
219			{
220			int i;
221
222			/* HTTP might've redirected and FTP handles wildcards, so
223			re-scan the URL for every conn */
224			for( i = 0; i < axel->conf->num_connections; i ++ )
225			{
226			conn_set( &axel->conn[i], axel->url->text );
227			axel->url = axel->url->next;
228			axel->conn[i].local_if = axel->conf->interfaces->text;
229			axel->conf->interfaces = axel->conf->interfaces->next;
230			axel->conn[i].conf = axel->conf;
231	appaji-guest	4	if( i ) axel->conn[i].supported = 1;
232	appaji-guest	2	}
233
234			if( axel->conf->verbose > 0 )
235			axel_message( axel, _("Starting download") );
236
237			for( i = 0; i < axel->conf->num_connections; i ++ )
238			if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
239			{
240			if( axel->conf->verbose >= 2 )
241			axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
242			i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
243			if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
244			{
245			axel_message( axel, _("pthread error!!!") );
246			axel->ready = -1;
247			}
248			else
249			{
250			axel->conn[i].last_transfer = gettime();
251			axel->conn[i].state = 1;
252			}
253			}
254
255			/* The real downloading will start now, so let's start counting */
256			axel->start_time = gettime();
257			axel->ready = 0;
258			}
259
260			/* Main 'loop' */
261			void axel_do( axel_t *axel )
262			{
263			fd_set fds[1];
264	phihag-guest	74	int hifd, i;
265			long long int remaining,size;
266	appaji-guest	2	struct timeval timeval[1];
267
268			/* Create statefile if necessary */
269			if( gettime() > axel->next_state )
270			{
271			save_state( axel );
272			axel->next_state = gettime() + axel->conf->save_state_interval;
273			}
274
275			/* Wait for data on (one of) the connections */
276			FD_ZERO( fds );
277			hifd = 0;
278			for( i = 0; i < axel->conf->num_connections; i ++ )
279			{
280			if( axel->conn[i].enabled )
281			FD_SET( axel->conn[i].fd, fds );
282			hifd = max( hifd, axel->conn[i].fd );
283			}
284			if( hifd == 0 )
285			{
286			/* No connections yet. Wait... */
287			usleep( 100000 );
288			goto conn_check;
289			}
290			else
291			{
292			timeval->tv_sec = 0;
293			timeval->tv_usec = 100000;
294			/* A select() error probably means it was interrupted
295			by a signal, or that something else's very wrong... */
296			if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
297			{
298			axel->ready = -1;
299			return;
300			}
301			}
302
303			/* Handle connections which need attention */
304			for( i = 0; i < axel->conf->num_connections; i ++ )
305			if( axel->conn[i].enabled ) {
306			if( FD_ISSET( axel->conn[i].fd, fds ) )
307			{
308			axel->conn[i].last_transfer = gettime();
309			size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
310			if( size == -1 )
311			{
312			if( axel->conf->verbose )
313			{
314			axel_message( axel, _("Error on connection %i! "
315			"Connection closed"), i );
316			}
317			axel->conn[i].enabled = 0;
318			conn_disconnect( &axel->conn[i] );
319			continue;
320			}
321			else if( size == 0 )
322			{
323			if( axel->conf->verbose )
324			{
325			/* Only abnormal behaviour if: */
326			if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
327			{
328			axel_message( axel, _("Connection %i unexpectedly closed"), i );
329			}
330			else
331			{
332			axel_message( axel, _("Connection %i finished"), i );
333			}
334			}
335			if( !axel->conn[0].supported )
336			{
337			axel->ready = 1;
338			}
339			axel->conn[i].enabled = 0;
340			conn_disconnect( &axel->conn[i] );
341			continue;
342			}
343	phihag-guest	74	/* remaining == Bytes to go */
344			remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
345			if( remaining < size )
346	appaji-guest	2	{
347			if( axel->conf->verbose )
348			{
349			axel_message( axel, _("Connection %i finished"), i );
350			}
351			axel->conn[i].enabled = 0;
352			conn_disconnect( &axel->conn[i] );
353	phihag-guest	74	size = remaining;
354	appaji-guest	2	/* Don't terminate, still stuff to write! */
355			}
356			/* This should always succeed.. */
357			lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
358			if( write( axel->outfd, buffer, size ) != size )
359			{
360
361			axel_message( axel, _("Write error!") );
362			axel->ready = -1;
363			return;
364			}
365			axel->conn[i].currentbyte += size;
366			axel->bytes_done += size;
367			}
368			else
369			{
370			if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
371			{
372			if( axel->conf->verbose )
373			axel_message( axel, _("Connection %i timed out"), i );
374			conn_disconnect( &axel->conn[i] );
375			axel->conn[i].enabled = 0;
376			}
377			} }
378
379			if( axel->ready )
380			return;
381
382			conn_check:
383			/* Look for aborted connections and attempt to restart them. */
384			for( i = 0; i < axel->conf->num_connections; i ++ )
385			{
386			if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
387			{
388			if( axel->conn[i].state == 0 )
389	phihag-guest	63	{
390			// Wait for termination of this thread
391			pthread_join(axel->conn[i].setup_thread, NULL);
392
393	appaji-guest	2	conn_set( &axel->conn[i], axel->url->text );
394			axel->url = axel->url->next;
395			/* axel->conn[i].local_if = axel->conf->interfaces->text;
396			axel->conf->interfaces = axel->conf->interfaces->next; */
397			if( axel->conf->verbose >= 2 )
398			axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
399			i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
400			if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
401			{
402			axel->conn[i].state = 1;
403			axel->conn[i].last_transfer = gettime();
404			}
405			else
406			{
407			axel_message( axel, _("pthread error!!!") );
408			axel->ready = -1;
409			}
410			}
411			else
412			{
413			if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
414			{
415			pthread_cancel( *axel->conn[i].setup_thread );
416			axel->conn[i].state = 0;
417			}
418			}
419			}
420			}
421
422			/* Calculate current average speed and finish_time */
423			axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
424			axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );
425
426			/* Check speed. If too high, delay for some time to slow things
427			down a bit. I think a 5% deviation should be acceptable. */
428			if( axel->conf->max_speed > 0 )
429			{
430			if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
431			axel->delay_time += 10000;
432			else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
433			axel->delay_time -= 10000;
434			else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
435			axel->delay_time = 0;
436			usleep( axel->delay_time );
437			}
438
439			/* Ready? */
440			if( axel->bytes_done == axel->size )
441			axel->ready = 1;
442			}
443
444			/* Close an axel connection */
445			void axel_close( axel_t *axel )
446			{
447			int i;
448			message_t *m;
449
450			/* Terminate any thread still running */
451			for( i = 0; i < axel->conf->num_connections; i ++ )
452	appaji-guest	6	/* don't try to kill non existing thread */
453			if ( *axel->conn[i].setup_thread != 0 )
454			pthread_cancel( *axel->conn[i].setup_thread );
455	appaji-guest	2
456			/* Delete state file if necessary */
457			if( axel->ready == 1 )
458			{
459			snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
460			unlink( buffer );
461			}
462			/* Else: Create it.. */
463			else if( axel->bytes_done > 0 )
464			{
465			save_state( axel );
466			}
467
468			/* Delete any message not processed yet */
469			while( axel->message )
470			{
471			m = axel->message;
472			axel->message = axel->message->next;
473			free( m );
474			}
475
476			/* Close all connections and local file */
477			close( axel->outfd );
478			for( i = 0; i < axel->conf->num_connections; i ++ )
479			conn_disconnect( &axel->conn[i] );
480
481			free( axel->conn );
482			free( axel );
483			}
484
485			/* time() with more precision */
486			double gettime()
487			{
488			struct timeval time[1];
489
490			gettimeofday( time, 0 );
491			return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
492			}
493
494			/* Save the state of the current download */
495			void save_state( axel_t *axel )
496			{
497			int fd, i;
498			char fn[MAX_STRING+4];
499
500			/* No use for such a file if the server doesn't support
501			resuming anyway.. */
502			if( !axel->conn[0].supported )
503			return;
504
505			snprintf( fn, MAX_STRING, "%s.st", axel->filename );
506			if( ( fd = open( fn, O_CREAT \| O_TRUNC \| O_WRONLY, 0666 ) ) == -1 )
507			{
508			return; /* Not 100% fatal.. */
509			}
510			write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
511			write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
512			for( i = 0; i < axel->conf->num_connections; i ++ )
513			{
514			write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
515			}
516			close( fd );
517			}
518
519			/* Thread used to set up a connection */
520			void setup_thread( void c )
521			{
522			conn_t *conn = c;
523			int oldstate;
524
525			/* Allow this thread to be killed at any time. */
526			pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
527			pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
528
529			if( conn_setup( conn ) )
530			{
531			conn->last_transfer = gettime();
532			if( conn_exec( conn ) )
533			{
534			conn->last_transfer = gettime();
535			conn->enabled = 1;
536			conn->state = 0;
537			return( NULL );
538			}
539			}
540
541			conn_disconnect( conn );
542			conn->state = 0;
543			return( NULL );
544			}
545
546			/* Add a message to the axel->message structure */
547			static void axel_message( axel_t axel, char format, ... )
548			{
549			message_t m = malloc( sizeof( message_t ) ), n = axel->message;
550			va_list params;
551
552			memset( m, 0, sizeof( message_t ) );
553			va_start( params, format );
554			vsnprintf( m->text, MAX_STRING, format, params );
555			va_end( params );
556
557			if( axel->message == NULL )
558			{
559			axel->message = m;
560			}
561			else
562			{
563			while( n->next != NULL )
564			n = n->next;
565			n->next = m;
566			}
567			}
568
569			/* Divide the file and set the locations for each connection */
570			static void axel_divide( axel_t *axel )
571			{
572			int i;
573
574			axel->conn[0].currentbyte = 0;
575			axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
576			for( i = 1; i < axel->conf->num_connections; i ++ )
577			{
578			#ifdef DEBUG
579	appaji-guest	25	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
580	appaji-guest	2	#endif
581			axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
582			axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
583			}
584			axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
585			#ifdef DEBUG
586	appaji-guest	25	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
587	appaji-guest	2	#endif
588			}