axel/trunk/axel.c

  /********************************************************************\
  * Axel -- A lighter download accelerator for Linux and other Unices. *
  *                                                                    *
  * Copyright 2001 Wilmer van der Gaast                                *
  \********************************************************************/

/* Main control                                                         */

/*
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License with
  the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
  if not, write to the Free Software Foundation, Inc., 59 Temple Place,
  Suite 330, Boston, MA  02111-1307  USA
*/

#include "axel.h"

/* Axel */
static void save_state( axel_t *axel );
static void *setup_thread( void * );
static void axel_message( axel_t *axel, char *format, ... );
static void axel_divide( axel_t *axel );

static char *buffer = NULL;

/* Create a new axel_t structure                                        */
axel_t *axel_new( conf_t *conf, int count, void *url )
{
        search_t *res;
        axel_t *axel;
        url_t *u;
        char *s;
        int i;
        
        axel = malloc( sizeof( axel_t ) );
        memset( axel, 0, sizeof( axel_t ) );
        *axel->conf = *conf;
        axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
        memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
                {
                        if( axel->conf->verbose >= 2 )
                                axel_message( axel, _("Buffer resized for this speed.") );
                        axel->conf->buffer_size = axel->conf->max_speed;
                }
                axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
        }
        if( buffer == NULL )
                buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
        
        if( count == 0 )
        {
                axel->url = malloc( sizeof( url_t ) );
                axel->url->next = axel->url;
                strncpy( axel->url->text, (char *) url, MAX_STRING );
        }
        else
        {
                res = (search_t *) url;
                u = axel->url = malloc( sizeof( url_t ) );
                for( i = 0; i < count; i ++ )
                {
                        strncpy( u->text, res[i].url, MAX_STRING );
                        if( i < count - 1 )
                        {
                                u->next = malloc( sizeof( url_t ) );
                                u = u->next;
                        }
                        else
                        {
                                u->next = axel->url;
                        }
                }
        }
        
        axel->conn[0].conf = axel->conf;
        if( !conn_set( &axel->conn[0], axel->url->text ) )
        {
                axel_message( axel, _("Could not parse URL.\n") );
                axel->ready = -1;
                return( axel );
        }

        axel->conn[0].local_if = axel->conf->interfaces->text;
        axel->conf->interfaces = axel->conf->interfaces->next;
        
        strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        http_decode( axel->filename );
        if( *axel->filename == 0 )      /* Index page == no fn          */
                strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
        if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
                *s = 0;         /* Get rid of CGI parameters            */
        
        if( !conn_init( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        
        /* This does more than just checking the file size, it all depends
           on the protocol used.                                        */
        if( !conn_info( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        s = conn_url( axel->conn );
        strncpy( axel->url->text, s, MAX_STRING );
        if( ( axel->size = axel->conn[0].size ) != INT_MAX )
        {
                if( axel->conf->verbose > 0 )
                        axel_message( axel, _("File size: %lld bytes"), axel->size );
        }
        
        /* Wildcards in URL --> Get complete filename                   */
        if( strchr( axel->filename, '*' ) || strchr( axel->filename, '?' ) )
                strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        
        return( axel );
}

/* Open a local file to store the downloaded data                       */
int axel_open( axel_t *axel )
{
        int i, fd;
        long long int j;
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Opening output file %s"), axel->filename );
        snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
        
        axel->outfd = -1;
        
        /* Check whether server knows about RESTart and switch back to
           single connection download if necessary                      */
        if( !axel->conn[0].supported )
        {
                axel_message( axel, _("Server unsupported, "
                        "starting from scratch with one connection.") );
                axel->conf->num_connections = 1;
                axel->conn = realloc( axel->conn, sizeof( conn_t ) );
                axel_divide( axel );
        }
        else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
        {
                read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
                
                axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
                memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );

                axel_divide( axel );
                
                read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
                for( i = 0; i < axel->conf->num_connections; i ++ )
                        read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );

                axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
                        axel->bytes_done, axel->size - axel->bytes_done );
                
                close( fd );
                
                if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
        }

        /* If outfd == -1 we have to start from scrath now              */
        if( axel->outfd == -1 )
        {
                axel_divide( axel );
                
                if( ( axel->outfd = open( axel->filename, O_CREAT | O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
                
                /* And check whether the filesystem can handle seeks to
                   past-EOF areas.. Speeds things up. :) AFAIK this
                   should just not happen:                              */
                if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
                {
                        /* But if the OS/fs does not allow to seek behind
                           EOF, we have to fill the file with zeroes before
                           starting. Slow..                             */
                        axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
                        lseek( axel->outfd, 0, SEEK_SET );
                        memset( buffer, 0, axel->conf->buffer_size );
                        j = axel->size;
                        while( j > 0 )
                        {
                                write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
                                j -= axel->conf->buffer_size;
                        }
                }
        }
        
        return( 1 );
}

/* Start downloading                                                    */
void axel_start( axel_t *axel )
{
        int i;
        
        /* HTTP might've redirected and FTP handles wildcards, so
           re-scan the URL for every conn                               */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                conn_set( &axel->conn[i], axel->url->text );
                axel->url = axel->url->next;
                axel->conn[i].local_if = axel->conf->interfaces->text;
                axel->conf->interfaces = axel->conf->interfaces->next;
                axel->conn[i].conf = axel->conf;
                if( i ) axel->conn[i].supported = 1;
        }
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Starting download") );
        
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
        {
                if( axel->conf->verbose >= 2 )
                {
                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                }
                
                axel->conn[i].state = 1;
                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
                {
                        axel_message( axel, _("pthread error!!!") );
                        axel->ready = -1;
                }
                else
                {
                        axel->conn[i].last_transfer = gettime();
                }
        }
        
        /* The real downloading will start now, so let's start counting */
        axel->start_time = gettime();
        axel->ready = 0;
}

/* Main 'loop'                                                          */
void axel_do( axel_t *axel )
{
        fd_set fds[1];
        int hifd, i;
        long long int remaining,size;
        struct timeval timeval[1];
        
        /* Create statefile if necessary                                */
        if( gettime() > axel->next_state )
        {
                save_state( axel );
                axel->next_state = gettime() + axel->conf->save_state_interval;
        }
        
        /* Wait for data on (one of) the connections                    */
        FD_ZERO( fds );
        hifd = 0;
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( axel->conn[i].enabled )
                        FD_SET( axel->conn[i].fd, fds );
                hifd = max( hifd, axel->conn[i].fd );
        }
        if( hifd == 0 )
        {
                /* No connections yet. Wait...                          */
                usleep( 100000 );
                goto conn_check;
        }
        else
        {
                timeval->tv_sec = 0;
                timeval->tv_usec = 100000;
                /* A select() error probably means it was interrupted
                   by a signal, or that something else's very wrong...  */
                if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
                {
                        axel->ready = -1;
                        return;
                }
        }
        
        /* Handle connections which need attention                      */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].enabled ) {
        if( FD_ISSET( axel->conn[i].fd, fds ) )
        {
                axel->conn[i].last_transfer = gettime();
                size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
                if( size == -1 )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Error on connection %i! "
                                        "Connection closed"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                else if( size == 0 )
                {
                        if( axel->conf->verbose )
                        {
                                /* Only abnormal behaviour if:          */
                                if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
                                {
                                        axel_message( axel, _("Connection %i unexpectedly closed"), i );
                                }
                                else
                                {
                                        axel_message( axel, _("Connection %i finished"), i );
                                }
                        }
                        if( !axel->conn[0].supported )
                        {
                                axel->ready = 1;
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                /* remaining == Bytes to go                                     */
                remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
                if( remaining < size )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Connection %i finished"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        size = remaining;
                        /* Don't terminate, still stuff to write!       */
                }
                /* This should always succeed..                         */
                lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
                if( write( axel->outfd, buffer, size ) != size )
                {
                        
                        axel_message( axel, _("Write error!") );
                        axel->ready = -1;
                        return;
                }
                axel->conn[i].currentbyte += size;
                axel->bytes_done += size;
        }
        else
        {
                if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
                {
                        if( axel->conf->verbose )
                                axel_message( axel, _("Connection %i timed out"), i );
                        conn_disconnect( &axel->conn[i] );
                        axel->conn[i].enabled = 0;
                }
        } }
        
        if( axel->ready )
                return;
        
conn_check:
        /* Look for aborted connections and attempt to restart them.    */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
                {
                        if( axel->conn[i].state == 0 )
                        {       
                                // Wait for termination of this thread
                                pthread_join(*(axel->conn[i].setup_thread), NULL);
                                
                                conn_set( &axel->conn[i], axel->url->text );
                                axel->url = axel->url->next;
                                /* axel->conn[i].local_if = axel->conf->interfaces->text;
                                axel->conf->interfaces = axel->conf->interfaces->next; */
                                if( axel->conf->verbose >= 2 )
                                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                                
                                axel->conn[i].state = 1;
                                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
                                {
                                        axel->conn[i].last_transfer = gettime();
                                }
                                else
                                {
                                        axel_message( axel, _("pthread error!!!") );
                                        axel->ready = -1;
                                }
                        }
                        else
                        {
                                if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
                                {
                                        pthread_cancel( *axel->conn[i].setup_thread );
                                        axel->conn[i].state = 0;
                                }
                        }
                }
        }

        /* Calculate current average speed and finish_time              */
        axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
        axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );

        /* Check speed. If too high, delay for some time to slow things
           down a bit. I think a 5% deviation should be acceptable.     */
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
                        axel->delay_time += 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
                        axel->delay_time -= 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
                        axel->delay_time = 0;
                usleep( axel->delay_time );
        }
        
        /* Ready?                                                       */
        if( axel->bytes_done == axel->size )
                axel->ready = 1;
}

/* Close an axel connection                                             */
void axel_close( axel_t *axel )
{
        int i;
        message_t *m;
        
        /* Terminate any thread still running                           */
        for( i = 0; i < axel->conf->num_connections; i ++ )
                /* don't try to kill non existing thread */
                if ( *axel->conn[i].setup_thread != 0 )
                        pthread_cancel( *axel->conn[i].setup_thread );
        
        /* Delete state file if necessary                               */
        if( axel->ready == 1 )
        {
                snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
                unlink( buffer );
        }
        /* Else: Create it..                                            */
        else if( axel->bytes_done > 0 )
        {
                save_state( axel );
        }

        /* Delete any message not processed yet                         */
        while( axel->message )
        {
                m = axel->message;
                axel->message = axel->message->next;
                free( m );
        }
        
        /* Close all connections and local file                         */
        close( axel->outfd );
        for( i = 0; i < axel->conf->num_connections; i ++ )
                conn_disconnect( &axel->conn[i] );

        free( axel->conn );
        free( axel );
}

/* time() with more precision                                           */
double gettime()
{
        struct timeval time[1];
        
        gettimeofday( time, 0 );
        return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
}

/* Save the state of the current download                               */
void save_state( axel_t *axel )
{
        int fd, i;
        char fn[MAX_STRING+4];

        /* No use for such a file if the server doesn't support
           resuming anyway..                                            */
        if( !axel->conn[0].supported )
                return;
        
        snprintf( fn, MAX_STRING, "%s.st", axel->filename );
        if( ( fd = open( fn, O_CREAT | O_TRUNC | O_WRONLY, 0666 ) ) == -1 )
        {
                return;         /* Not 100% fatal..                     */
        }
        write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
        write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
        }
        close( fd );
}

/* Thread used to set up a connection                                   */
void *setup_thread( void *c )
{
        conn_t *conn = c;
        int oldstate;
        
        /* Allow this thread to be killed at any time.                  */
        pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
        pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
        
        if( conn_setup( conn ) )
        {
                conn->last_transfer = gettime();
                if( conn_exec( conn ) )
                {
                        conn->last_transfer = gettime();
                        conn->enabled = 1;
                        conn->state = 0;
                        return( NULL );
                }
        }
        
        conn_disconnect( conn );
        conn->state = 0;
        return( NULL );
}

/* Add a message to the axel->message structure                         */
static void axel_message( axel_t *axel, char *format, ... )
{
        message_t *m = malloc( sizeof( message_t ) ), *n = axel->message;
        va_list params;
        
        memset( m, 0, sizeof( message_t ) );
        va_start( params, format );
        vsnprintf( m->text, MAX_STRING, format, params );
        va_end( params );
        
        if( axel->message == NULL )
        {
                axel->message = m;
        }
        else
        {
                while( n->next != NULL )
                        n = n->next;
                n->next = m;
        }
}

/* Divide the file and set the locations for each connection            */
static void axel_divide( axel_t *axel )
{
        int i;
        
        axel->conn[0].currentbyte = 0;
        axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
        for( i = 1; i < axel->conf->num_connections; i ++ )
        {
#ifdef DEBUG
                printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
                axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
                axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
        }
        axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
#ifdef DEBUG
        printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
}
1	appaji-guest	2	/********************************************************************\
2			* Axel -- A lighter download accelerator for Linux and other Unices. *
3			* *
4			* Copyright 2001 Wilmer van der Gaast *
5			\********************************************************************/
6
7			/* Main control */
8
9			/*
10			This program is free software; you can redistribute it and/or modify
11			it under the terms of the GNU General Public License as published by
12			the Free Software Foundation; either version 2 of the License, or
13			(at your option) any later version.
14
15			This program is distributed in the hope that it will be useful,
16			but WITHOUT ANY WARRANTY; without even the implied warranty of
17			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			GNU General Public License for more details.
19
20			You should have received a copy of the GNU General Public License with
21			the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
22			if not, write to the Free Software Foundation, Inc., 59 Temple Place,
23			Suite 330, Boston, MA 02111-1307 USA
24			*/
25
26			#include "axel.h"
27
28			/* Axel */
29			static void save_state( axel_t *axel );
30			static void setup_thread( void );
31			static void axel_message( axel_t axel, char format, ... );
32			static void axel_divide( axel_t *axel );
33
34			static char *buffer = NULL;
35
36			/* Create a new axel_t structure */
37			axel_t axel_new( conf_t conf, int count, void *url )
38			{
39			search_t *res;
40			axel_t *axel;
41			url_t *u;
42			char *s;
43			int i;
44
45			axel = malloc( sizeof( axel_t ) );
46			memset( axel, 0, sizeof( axel_t ) );
47			axel->conf = conf;
48			axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
49			memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
50			if( axel->conf->max_speed > 0 )
51			{
52			if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
53			{
54			if( axel->conf->verbose >= 2 )
55			axel_message( axel, _("Buffer resized for this speed.") );
56			axel->conf->buffer_size = axel->conf->max_speed;
57			}
58			axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
59			}
60			if( buffer == NULL )
61			buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
62
63			if( count == 0 )
64			{
65			axel->url = malloc( sizeof( url_t ) );
66			axel->url->next = axel->url;
67	appaji-guest	14	strncpy( axel->url->text, (char *) url, MAX_STRING );
68	appaji-guest	2	}
69			else
70			{
71			res = (search_t *) url;
72			u = axel->url = malloc( sizeof( url_t ) );
73			for( i = 0; i < count; i ++ )
74			{
75	appaji-guest	14	strncpy( u->text, res[i].url, MAX_STRING );
76	appaji-guest	2	if( i < count - 1 )
77			{
78			u->next = malloc( sizeof( url_t ) );
79			u = u->next;
80			}
81			else
82			{
83			u->next = axel->url;
84			}
85			}
86			}
87
88			axel->conn[0].conf = axel->conf;
89			if( !conn_set( &axel->conn[0], axel->url->text ) )
90			{
91			axel_message( axel, _("Could not parse URL.\n") );
92			axel->ready = -1;
93			return( axel );
94			}
95
96			axel->conn[0].local_if = axel->conf->interfaces->text;
97			axel->conf->interfaces = axel->conf->interfaces->next;
98
99	appaji-guest	14	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
100	appaji-guest	2	http_decode( axel->filename );
101			if( axel->filename == 0 ) / Index page == no fn */
102	appaji-guest	14	strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
103	appaji-guest	2	if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
104			s = 0; / Get rid of CGI parameters */
105
106			if( !conn_init( &axel->conn[0] ) )
107			{
108			axel_message( axel, axel->conn[0].message );
109			axel->ready = -1;
110			return( axel );
111			}
112
113			/* This does more than just checking the file size, it all depends
114			on the protocol used. */
115			if( !conn_info( &axel->conn[0] ) )
116			{
117			axel_message( axel, axel->conn[0].message );
118			axel->ready = -1;
119			return( axel );
120			}
121			s = conn_url( axel->conn );
122	appaji-guest	14	strncpy( axel->url->text, s, MAX_STRING );
123	appaji-guest	2	if( ( axel->size = axel->conn[0].size ) != INT_MAX )
124			{
125			if( axel->conf->verbose > 0 )
126	appaji-guest	25	axel_message( axel, _("File size: %lld bytes"), axel->size );
127	appaji-guest	2	}
128
129			/* Wildcards in URL --> Get complete filename */
130			if( strchr( axel->filename, '*' ) \|\| strchr( axel->filename, '?' ) )
131	appaji-guest	14	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
132	appaji-guest	2
133			return( axel );
134			}
135
136			/* Open a local file to store the downloaded data */
137			int axel_open( axel_t *axel )
138			{
139			int i, fd;
140	appaji-guest	25	long long int j;
141	appaji-guest	2
142			if( axel->conf->verbose > 0 )
143			axel_message( axel, _("Opening output file %s"), axel->filename );
144			snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
145
146			axel->outfd = -1;
147
148			/* Check whether server knows about RESTart and switch back to
149			single connection download if necessary */
150			if( !axel->conn[0].supported )
151			{
152			axel_message( axel, _("Server unsupported, "
153			"starting from scratch with one connection.") );
154			axel->conf->num_connections = 1;
155			axel->conn = realloc( axel->conn, sizeof( conn_t ) );
156			axel_divide( axel );
157			}
158			else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
159			{
160			read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
161
162			axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
163			memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );
164
165			axel_divide( axel );
166
167			read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
168			for( i = 0; i < axel->conf->num_connections; i ++ )
169			read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
170
171	appaji-guest	25	axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
172	appaji-guest	2	axel->bytes_done, axel->size - axel->bytes_done );
173
174			close( fd );
175
176			if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
177			{
178			axel_message( axel, _("Error opening local file") );
179			return( 0 );
180			}
181			}
182
183			/* If outfd == -1 we have to start from scrath now */
184			if( axel->outfd == -1 )
185			{
186			axel_divide( axel );
187
188			if( ( axel->outfd = open( axel->filename, O_CREAT \| O_WRONLY, 0666 ) ) == -1 )
189			{
190			axel_message( axel, _("Error opening local file") );
191			return( 0 );
192			}
193
194			/* And check whether the filesystem can handle seeks to
195			past-EOF areas.. Speeds things up. :) AFAIK this
196			should just not happen: */
197	appaji-guest	26	if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
198	appaji-guest	2	{
199			/* But if the OS/fs does not allow to seek behind
200			EOF, we have to fill the file with zeroes before
201			starting. Slow.. */
202			axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
203			lseek( axel->outfd, 0, SEEK_SET );
204			memset( buffer, 0, axel->conf->buffer_size );
205	appaji-guest	25	j = axel->size;
206			while( j > 0 )
207	appaji-guest	2	{
208	appaji-guest	25	write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
209			j -= axel->conf->buffer_size;
210	appaji-guest	2	}
211			}
212			}
213
214			return( 1 );
215			}
216
217			/* Start downloading */
218			void axel_start( axel_t *axel )
219			{
220			int i;
221
222			/* HTTP might've redirected and FTP handles wildcards, so
223			re-scan the URL for every conn */
224			for( i = 0; i < axel->conf->num_connections; i ++ )
225			{
226			conn_set( &axel->conn[i], axel->url->text );
227			axel->url = axel->url->next;
228			axel->conn[i].local_if = axel->conf->interfaces->text;
229			axel->conf->interfaces = axel->conf->interfaces->next;
230			axel->conn[i].conf = axel->conf;
231	appaji-guest	4	if( i ) axel->conn[i].supported = 1;
232	appaji-guest	2	}
233
234			if( axel->conf->verbose > 0 )
235			axel_message( axel, _("Starting download") );
236
237			for( i = 0; i < axel->conf->num_connections; i ++ )
238			if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
239			{
240			if( axel->conf->verbose >= 2 )
241	phihag-guest	94	{
242	appaji-guest	2	axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
243			i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
244	phihag-guest	94	}
245
246			axel->conn[i].state = 1;
247	appaji-guest	2	if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
248			{
249			axel_message( axel, _("pthread error!!!") );
250			axel->ready = -1;
251			}
252			else
253			{
254			axel->conn[i].last_transfer = gettime();
255			}
256			}
257
258			/* The real downloading will start now, so let's start counting */
259			axel->start_time = gettime();
260			axel->ready = 0;
261			}
262
263			/* Main 'loop' */
264			void axel_do( axel_t *axel )
265			{
266			fd_set fds[1];
267	phihag-guest	74	int hifd, i;
268			long long int remaining,size;
269	appaji-guest	2	struct timeval timeval[1];
270
271			/* Create statefile if necessary */
272			if( gettime() > axel->next_state )
273			{
274			save_state( axel );
275			axel->next_state = gettime() + axel->conf->save_state_interval;
276			}
277
278			/* Wait for data on (one of) the connections */
279			FD_ZERO( fds );
280			hifd = 0;
281			for( i = 0; i < axel->conf->num_connections; i ++ )
282			{
283			if( axel->conn[i].enabled )
284			FD_SET( axel->conn[i].fd, fds );
285			hifd = max( hifd, axel->conn[i].fd );
286			}
287			if( hifd == 0 )
288			{
289			/* No connections yet. Wait... */
290			usleep( 100000 );
291			goto conn_check;
292			}
293			else
294			{
295			timeval->tv_sec = 0;
296			timeval->tv_usec = 100000;
297			/* A select() error probably means it was interrupted
298			by a signal, or that something else's very wrong... */
299			if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
300			{
301			axel->ready = -1;
302			return;
303			}
304			}
305
306			/* Handle connections which need attention */
307			for( i = 0; i < axel->conf->num_connections; i ++ )
308			if( axel->conn[i].enabled ) {
309			if( FD_ISSET( axel->conn[i].fd, fds ) )
310			{
311			axel->conn[i].last_transfer = gettime();
312			size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
313			if( size == -1 )
314			{
315			if( axel->conf->verbose )
316			{
317			axel_message( axel, _("Error on connection %i! "
318			"Connection closed"), i );
319			}
320			axel->conn[i].enabled = 0;
321			conn_disconnect( &axel->conn[i] );
322			continue;
323			}
324			else if( size == 0 )
325			{
326			if( axel->conf->verbose )
327			{
328			/* Only abnormal behaviour if: */
329			if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
330			{
331			axel_message( axel, _("Connection %i unexpectedly closed"), i );
332			}
333			else
334			{
335			axel_message( axel, _("Connection %i finished"), i );
336			}
337			}
338			if( !axel->conn[0].supported )
339			{
340			axel->ready = 1;
341			}
342			axel->conn[i].enabled = 0;
343			conn_disconnect( &axel->conn[i] );
344			continue;
345			}
346	phihag-guest	74	/* remaining == Bytes to go */
347			remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
348			if( remaining < size )
349	appaji-guest	2	{
350			if( axel->conf->verbose )
351			{
352			axel_message( axel, _("Connection %i finished"), i );
353			}
354			axel->conn[i].enabled = 0;
355			conn_disconnect( &axel->conn[i] );
356	phihag-guest	74	size = remaining;
357	appaji-guest	2	/* Don't terminate, still stuff to write! */
358			}
359			/* This should always succeed.. */
360			lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
361			if( write( axel->outfd, buffer, size ) != size )
362			{
363
364			axel_message( axel, _("Write error!") );
365			axel->ready = -1;
366			return;
367			}
368			axel->conn[i].currentbyte += size;
369			axel->bytes_done += size;
370			}
371			else
372			{
373			if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
374			{
375			if( axel->conf->verbose )
376			axel_message( axel, _("Connection %i timed out"), i );
377			conn_disconnect( &axel->conn[i] );
378			axel->conn[i].enabled = 0;
379			}
380			} }
381
382			if( axel->ready )
383			return;
384
385			conn_check:
386			/* Look for aborted connections and attempt to restart them. */
387			for( i = 0; i < axel->conf->num_connections; i ++ )
388			{
389			if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
390			{
391			if( axel->conn[i].state == 0 )
392	phihag-guest	63	{
393			// Wait for termination of this thread
394	phihag-guest	82	pthread_join(*(axel->conn[i].setup_thread), NULL);
395	phihag-guest	63
396	appaji-guest	2	conn_set( &axel->conn[i], axel->url->text );
397			axel->url = axel->url->next;
398			/* axel->conn[i].local_if = axel->conf->interfaces->text;
399			axel->conf->interfaces = axel->conf->interfaces->next; */
400			if( axel->conf->verbose >= 2 )
401			axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
402			i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
403	phihag-guest	94
404			axel->conn[i].state = 1;
405	appaji-guest	2	if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
406			{
407			axel->conn[i].last_transfer = gettime();
408			}
409			else
410			{
411			axel_message( axel, _("pthread error!!!") );
412			axel->ready = -1;
413			}
414			}
415			else
416			{
417			if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
418			{
419			pthread_cancel( *axel->conn[i].setup_thread );
420			axel->conn[i].state = 0;
421			}
422			}
423			}
424			}
425
426			/* Calculate current average speed and finish_time */
427			axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
428			axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );
429
430			/* Check speed. If too high, delay for some time to slow things
431			down a bit. I think a 5% deviation should be acceptable. */
432			if( axel->conf->max_speed > 0 )
433			{
434			if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
435			axel->delay_time += 10000;
436			else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
437			axel->delay_time -= 10000;
438			else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
439			axel->delay_time = 0;
440			usleep( axel->delay_time );
441			}
442
443			/* Ready? */
444			if( axel->bytes_done == axel->size )
445			axel->ready = 1;
446			}
447
448			/* Close an axel connection */
449			void axel_close( axel_t *axel )
450			{
451			int i;
452			message_t *m;
453
454			/* Terminate any thread still running */
455			for( i = 0; i < axel->conf->num_connections; i ++ )
456	appaji-guest	6	/* don't try to kill non existing thread */
457			if ( *axel->conn[i].setup_thread != 0 )
458			pthread_cancel( *axel->conn[i].setup_thread );
459	appaji-guest	2
460			/* Delete state file if necessary */
461			if( axel->ready == 1 )
462			{
463			snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
464			unlink( buffer );
465			}
466			/* Else: Create it.. */
467			else if( axel->bytes_done > 0 )
468			{
469			save_state( axel );
470			}
471
472			/* Delete any message not processed yet */
473			while( axel->message )
474			{
475			m = axel->message;
476			axel->message = axel->message->next;
477			free( m );
478			}
479
480			/* Close all connections and local file */
481			close( axel->outfd );
482			for( i = 0; i < axel->conf->num_connections; i ++ )
483			conn_disconnect( &axel->conn[i] );
484
485			free( axel->conn );
486			free( axel );
487			}
488
489			/* time() with more precision */
490			double gettime()
491			{
492			struct timeval time[1];
493
494			gettimeofday( time, 0 );
495			return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
496			}
497
498			/* Save the state of the current download */
499			void save_state( axel_t *axel )
500			{
501			int fd, i;
502			char fn[MAX_STRING+4];
503
504			/* No use for such a file if the server doesn't support
505			resuming anyway.. */
506			if( !axel->conn[0].supported )
507			return;
508
509			snprintf( fn, MAX_STRING, "%s.st", axel->filename );
510			if( ( fd = open( fn, O_CREAT \| O_TRUNC \| O_WRONLY, 0666 ) ) == -1 )
511			{
512			return; /* Not 100% fatal.. */
513			}
514			write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
515			write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
516			for( i = 0; i < axel->conf->num_connections; i ++ )
517			{
518			write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
519			}
520			close( fd );
521			}
522
523			/* Thread used to set up a connection */
524			void setup_thread( void c )
525			{
526			conn_t *conn = c;
527			int oldstate;
528
529			/* Allow this thread to be killed at any time. */
530			pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
531			pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
532
533			if( conn_setup( conn ) )
534			{
535			conn->last_transfer = gettime();
536			if( conn_exec( conn ) )
537			{
538			conn->last_transfer = gettime();
539			conn->enabled = 1;
540			conn->state = 0;
541			return( NULL );
542			}
543			}
544
545			conn_disconnect( conn );
546			conn->state = 0;
547			return( NULL );
548			}
549
550			/* Add a message to the axel->message structure */
551			static void axel_message( axel_t axel, char format, ... )
552			{
553			message_t m = malloc( sizeof( message_t ) ), n = axel->message;
554			va_list params;
555
556			memset( m, 0, sizeof( message_t ) );
557			va_start( params, format );
558			vsnprintf( m->text, MAX_STRING, format, params );
559			va_end( params );
560
561			if( axel->message == NULL )
562			{
563			axel->message = m;
564			}
565			else
566			{
567			while( n->next != NULL )
568			n = n->next;
569			n->next = m;
570			}
571			}
572
573			/* Divide the file and set the locations for each connection */
574			static void axel_divide( axel_t *axel )
575			{
576			int i;
577
578			axel->conn[0].currentbyte = 0;
579			axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
580			for( i = 1; i < axel->conf->num_connections; i ++ )
581			{
582			#ifdef DEBUG
583	appaji-guest	25	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
584	appaji-guest	2	#endif
585			axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
586			axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
587			}
588			axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
589			#ifdef DEBUG
590	appaji-guest	25	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
591	appaji-guest	2	#endif
592			}