axel/trunk/axel.c

  /********************************************************************\
  * Axel -- A lighter download accelerator for Linux and other Unices. *
  *                                                                    *
  * Copyright 2001 Wilmer van der Gaast                                *
  \********************************************************************/

/* Main control                                                         */

/*
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License with
  the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
  if not, write to the Free Software Foundation, Inc., 59 Temple Place,
  Suite 330, Boston, MA  02111-1307  USA
*/

#include "axel.h"

/* Axel */
static void save_state( axel_t *axel );
static void *setup_thread( void * );
static void axel_message( axel_t *axel, char *format, ... );
static void axel_divide( axel_t *axel );

static char *buffer = NULL;

/* Create a new axel_t structure                                        */
axel_t *axel_new( conf_t *conf, int count, void *url )
{
        search_t *res;
        axel_t *axel;
        url_t *u;
        char *s;
        int i;
        
        axel = malloc( sizeof( axel_t ) );
        memset( axel, 0, sizeof( axel_t ) );
        *axel->conf = *conf;
        axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
        memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
                {
                        if( axel->conf->verbose >= 2 )
                                axel_message( axel, _("Buffer resized for this speed.") );
                        axel->conf->buffer_size = axel->conf->max_speed;
                }
                axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
        }
        if( buffer == NULL )
                buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
        
        if( count == 0 )
        {
                axel->url = malloc( sizeof( url_t ) );
                axel->url->next = axel->url;
                strncpy( axel->url->text, (char *) url, MAX_STRING );
        }
        else
        {
                res = (search_t *) url;
                u = axel->url = malloc( sizeof( url_t ) );
                for( i = 0; i < count; i ++ )
                {
                        strncpy( u->text, res[i].url, MAX_STRING );
                        if( i < count - 1 )
                        {
                                u->next = malloc( sizeof( url_t ) );
                                u = u->next;
                        }
                        else
                        {
                                u->next = axel->url;
                        }
                }
        }
        
        axel->conn[0].conf = axel->conf;
        if( !conn_set( &axel->conn[0], axel->url->text ) )
        {
                axel_message( axel, _("Could not parse URL.\n") );
                axel->ready = -1;
                return( axel );
        }

        axel->conn[0].local_if = axel->conf->interfaces->text;
        axel->conf->interfaces = axel->conf->interfaces->next;
        
        strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        http_decode( axel->filename );
        if( *axel->filename == 0 )      /* Index page == no fn          */
                strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
        if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
                *s = 0;         /* Get rid of CGI parameters            */
        
        if( !conn_init( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        
        /* This does more than just checking the file size, it all depends
           on the protocol used.                                        */
        if( !conn_info( &axel->conn[0] ) )
        {
                axel_message( axel, axel->conn[0].message );
                axel->ready = -1;
                return( axel );
        }
        s = conn_url( axel->conn );
        strncpy( axel->url->text, s, MAX_STRING );
        if( ( axel->size = axel->conn[0].size ) != INT_MAX )
        {
                if( axel->conf->verbose > 0 )
                        axel_message( axel, _("File size: %lld bytes"), axel->size );
        }
        
        /* Wildcards in URL --> Get complete filename                   */
        if( strchr( axel->filename, '*' ) || strchr( axel->filename, '?' ) )
                strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
        
        return( axel );
}

/* Open a local file to store the downloaded data                       */
int axel_open( axel_t *axel )
{
        int i, fd;
        long long int j;
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Opening output file %s"), axel->filename );
        snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
        
        axel->outfd = -1;
        
        /* Check whether server knows about RESTart and switch back to
           single connection download if necessary                      */
        if( !axel->conn[0].supported )
        {
                axel_message( axel, _("Server unsupported, "
                        "starting from scratch with one connection.") );
                axel->conf->num_connections = 1;
                axel->conn = realloc( axel->conn, sizeof( conn_t ) );
                axel_divide( axel );
        }
        else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
        {
                read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
                
                axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
                memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );

                axel_divide( axel );
                
                read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
                for( i = 0; i < axel->conf->num_connections; i ++ )
                        read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );

                axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
                        axel->bytes_done, axel->size - axel->bytes_done );
                
                close( fd );
                
                if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
        }

        /* If outfd == -1 we have to start from scrath now              */
        if( axel->outfd == -1 )
        {
                axel_divide( axel );
                
                if( ( axel->outfd = open( axel->filename, O_CREAT | O_WRONLY, 0666 ) ) == -1 )
                {
                        axel_message( axel, _("Error opening local file") );
                        return( 0 );
                }
                
                /* And check whether the filesystem can handle seeks to
                   past-EOF areas.. Speeds things up. :) AFAIK this
                   should just not happen:                              */
                if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
                {
                        /* But if the OS/fs does not allow to seek behind
                           EOF, we have to fill the file with zeroes before
                           starting. Slow..                             */
                        axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
                        lseek( axel->outfd, 0, SEEK_SET );
                        memset( buffer, 0, axel->conf->buffer_size );
                        j = axel->size;
                        while( j > 0 )
                        {
                                write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
                                j -= axel->conf->buffer_size;
                        }
                }
        }
        
        return( 1 );
}

/* Start downloading                                                    */
void axel_start( axel_t *axel )
{
        int i;
        
        /* HTTP might've redirected and FTP handles wildcards, so
           re-scan the URL for every conn                               */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                conn_set( &axel->conn[i], axel->url->text );
                axel->url = axel->url->next;
                axel->conn[i].local_if = axel->conf->interfaces->text;
                axel->conf->interfaces = axel->conf->interfaces->next;
                axel->conn[i].conf = axel->conf;
                if( i ) axel->conn[i].supported = 1;
        }
        
        if( axel->conf->verbose > 0 )
                axel_message( axel, _("Starting download") );
        
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
        {
                if( axel->conf->verbose >= 2 )
                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
                {
                        axel_message( axel, _("pthread error!!!") );
                        axel->ready = -1;
                }
                else
                {
                        axel->conn[i].last_transfer = gettime();
                        axel->conn[i].state = 1;
                }
        }
        
        /* The real downloading will start now, so let's start counting */
        axel->start_time = gettime();
        axel->ready = 0;
}

/* Main 'loop'                                                          */
void axel_do( axel_t *axel )
{
        fd_set fds[1];
        int hifd, i;
        long long int remaining,size;
        struct timeval timeval[1];
        
        /* Create statefile if necessary                                */
        if( gettime() > axel->next_state )
        {
                save_state( axel );
                axel->next_state = gettime() + axel->conf->save_state_interval;
        }
        
        /* Wait for data on (one of) the connections                    */
        FD_ZERO( fds );
        hifd = 0;
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( axel->conn[i].enabled )
                        FD_SET( axel->conn[i].fd, fds );
                hifd = max( hifd, axel->conn[i].fd );
        }
        if( hifd == 0 )
        {
                /* No connections yet. Wait...                          */
                usleep( 100000 );
                goto conn_check;
        }
        else
        {
                timeval->tv_sec = 0;
                timeval->tv_usec = 100000;
                /* A select() error probably means it was interrupted
                   by a signal, or that something else's very wrong...  */
                if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
                {
                        axel->ready = -1;
                        return;
                }
        }
        
        /* Handle connections which need attention                      */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        if( axel->conn[i].enabled ) {
        if( FD_ISSET( axel->conn[i].fd, fds ) )
        {
                axel->conn[i].last_transfer = gettime();
                size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
                if( size == -1 )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Error on connection %i! "
                                        "Connection closed"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                else if( size == 0 )
                {
                        if( axel->conf->verbose )
                        {
                                /* Only abnormal behaviour if:          */
                                if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
                                {
                                        axel_message( axel, _("Connection %i unexpectedly closed"), i );
                                }
                                else
                                {
                                        axel_message( axel, _("Connection %i finished"), i );
                                }
                        }
                        if( !axel->conn[0].supported )
                        {
                                axel->ready = 1;
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        continue;
                }
                /* remaining == Bytes to go                                     */
                remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
                if( remaining < size )
                {
                        if( axel->conf->verbose )
                        {
                                axel_message( axel, _("Connection %i finished"), i );
                        }
                        axel->conn[i].enabled = 0;
                        conn_disconnect( &axel->conn[i] );
                        size = remaining;
                        /* Don't terminate, still stuff to write!       */
                }
                /* This should always succeed..                         */
                lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
                if( write( axel->outfd, buffer, size ) != size )
                {
                        
                        axel_message( axel, _("Write error!") );
                        axel->ready = -1;
                        return;
                }
                axel->conn[i].currentbyte += size;
                axel->bytes_done += size;
        }
        else
        {
                if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
                {
                        if( axel->conf->verbose )
                                axel_message( axel, _("Connection %i timed out"), i );
                        conn_disconnect( &axel->conn[i] );
                        axel->conn[i].enabled = 0;
                }
        } }
        
        if( axel->ready )
                return;
        
conn_check:
        /* Look for aborted connections and attempt to restart them.    */
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
                {
                        if( axel->conn[i].state == 0 )
                        {       
                                // Wait for termination of this thread
                                pthread_join(*(axel->conn[i].setup_thread), NULL);
                                
                                conn_set( &axel->conn[i], axel->url->text );
                                axel->url = axel->url->next;
                                /* axel->conn[i].local_if = axel->conf->interfaces->text;
                                axel->conf->interfaces = axel->conf->interfaces->next; */
                                if( axel->conf->verbose >= 2 )
                                        axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
                                                      i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
                                if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
                                {
                                        axel->conn[i].state = 1;
                                        axel->conn[i].last_transfer = gettime();
                                }
                                else
                                {
                                        axel_message( axel, _("pthread error!!!") );
                                        axel->ready = -1;
                                }
                        }
                        else
                        {
                                if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
                                {
                                        pthread_cancel( *axel->conn[i].setup_thread );
                                        axel->conn[i].state = 0;
                                }
                        }
                }
        }

        /* Calculate current average speed and finish_time              */
        axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
        axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );

        /* Check speed. If too high, delay for some time to slow things
           down a bit. I think a 5% deviation should be acceptable.     */
        if( axel->conf->max_speed > 0 )
        {
                if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
                        axel->delay_time += 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
                        axel->delay_time -= 10000;
                else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
                        axel->delay_time = 0;
                usleep( axel->delay_time );
        }
        
        /* Ready?                                                       */
        if( axel->bytes_done == axel->size )
                axel->ready = 1;
}

/* Close an axel connection                                             */
void axel_close( axel_t *axel )
{
        int i;
        message_t *m;
        
        /* Terminate any thread still running                           */
        for( i = 0; i < axel->conf->num_connections; i ++ )
                /* don't try to kill non existing thread */
                if ( *axel->conn[i].setup_thread != 0 )
                        pthread_cancel( *axel->conn[i].setup_thread );
        
        /* Delete state file if necessary                               */
        if( axel->ready == 1 )
        {
                snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
                unlink( buffer );
        }
        /* Else: Create it..                                            */
        else if( axel->bytes_done > 0 )
        {
                save_state( axel );
        }

        /* Delete any message not processed yet                         */
        while( axel->message )
        {
                m = axel->message;
                axel->message = axel->message->next;
                free( m );
        }
        
        /* Close all connections and local file                         */
        close( axel->outfd );
        for( i = 0; i < axel->conf->num_connections; i ++ )
                conn_disconnect( &axel->conn[i] );

        free( axel->conn );
        free( axel );
}

/* time() with more precision                                           */
double gettime()
{
        struct timeval time[1];
        
        gettimeofday( time, 0 );
        return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
}

/* Save the state of the current download                               */
void save_state( axel_t *axel )
{
        int fd, i;
        char fn[MAX_STRING+4];

        /* No use for such a file if the server doesn't support
           resuming anyway..                                            */
        if( !axel->conn[0].supported )
                return;
        
        snprintf( fn, MAX_STRING, "%s.st", axel->filename );
        if( ( fd = open( fn, O_CREAT | O_TRUNC | O_WRONLY, 0666 ) ) == -1 )
        {
                return;         /* Not 100% fatal..                     */
        }
        write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
        write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
        for( i = 0; i < axel->conf->num_connections; i ++ )
        {
                write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
        }
        close( fd );
}

/* Thread used to set up a connection                                   */
void *setup_thread( void *c )
{
        conn_t *conn = c;
        int oldstate;
        
        /* Allow this thread to be killed at any time.                  */
        pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
        pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
        
        if( conn_setup( conn ) )
        {
                conn->last_transfer = gettime();
                if( conn_exec( conn ) )
                {
                        conn->last_transfer = gettime();
                        conn->enabled = 1;
                        conn->state = 0;
                        return( NULL );
                }
        }
        
        conn_disconnect( conn );
        conn->state = 0;
        return( NULL );
}

/* Add a message to the axel->message structure                         */
static void axel_message( axel_t *axel, char *format, ... )
{
        message_t *m = malloc( sizeof( message_t ) ), *n = axel->message;
        va_list params;
        
        memset( m, 0, sizeof( message_t ) );
        va_start( params, format );
        vsnprintf( m->text, MAX_STRING, format, params );
        va_end( params );
        
        if( axel->message == NULL )
        {
                axel->message = m;
        }
        else
        {
                while( n->next != NULL )
                        n = n->next;
                n->next = m;
        }
}

/* Divide the file and set the locations for each connection            */
static void axel_divide( axel_t *axel )
{
        int i;
        
        axel->conn[0].currentbyte = 0;
        axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
        for( i = 1; i < axel->conf->num_connections; i ++ )
        {
#ifdef DEBUG
                printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
                axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
                axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
        }
        axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
#ifdef DEBUG
        printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
#endif
}
1	/********************************************************************\
2	* Axel -- A lighter download accelerator for Linux and other Unices. *
3	* *
4	* Copyright 2001 Wilmer van der Gaast *
5	\********************************************************************/
6
7	/* Main control */
8
9	/*
10	This program is free software; you can redistribute it and/or modify
11	it under the terms of the GNU General Public License as published by
12	the Free Software Foundation; either version 2 of the License, or
13	(at your option) any later version.
14
15	This program is distributed in the hope that it will be useful,
16	but WITHOUT ANY WARRANTY; without even the implied warranty of
17	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	GNU General Public License for more details.
19
20	You should have received a copy of the GNU General Public License with
21	the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
22	if not, write to the Free Software Foundation, Inc., 59 Temple Place,
23	Suite 330, Boston, MA 02111-1307 USA
24	*/
25
26	#include "axel.h"
27
28	/* Axel */
29	static void save_state( axel_t *axel );
30	static void setup_thread( void );
31	static void axel_message( axel_t axel, char format, ... );
32	static void axel_divide( axel_t *axel );
33
34	static char *buffer = NULL;
35
36	/* Create a new axel_t structure */
37	axel_t axel_new( conf_t conf, int count, void *url )
38	{
39	search_t *res;
40	axel_t *axel;
41	url_t *u;
42	char *s;
43	int i;
44
45	axel = malloc( sizeof( axel_t ) );
46	memset( axel, 0, sizeof( axel_t ) );
47	axel->conf = conf;
48	axel->conn = malloc( sizeof( conn_t ) * axel->conf->num_connections );
49	memset( axel->conn, 0, sizeof( conn_t ) * axel->conf->num_connections );
50	if( axel->conf->max_speed > 0 )
51	{
52	if( (float) axel->conf->max_speed / axel->conf->buffer_size < 0.5 )
53	{
54	if( axel->conf->verbose >= 2 )
55	axel_message( axel, _("Buffer resized for this speed.") );
56	axel->conf->buffer_size = axel->conf->max_speed;
57	}
58	axel->delay_time = (int) ( (float) 1000000 / axel->conf->max_speed * axel->conf->buffer_size * axel->conf->num_connections );
59	}
60	if( buffer == NULL )
61	buffer = malloc( max( MAX_STRING, axel->conf->buffer_size ) );
62
63	if( count == 0 )
64	{
65	axel->url = malloc( sizeof( url_t ) );
66	axel->url->next = axel->url;
67	strncpy( axel->url->text, (char *) url, MAX_STRING );
68	}
69	else
70	{
71	res = (search_t *) url;
72	u = axel->url = malloc( sizeof( url_t ) );
73	for( i = 0; i < count; i ++ )
74	{
75	strncpy( u->text, res[i].url, MAX_STRING );
76	if( i < count - 1 )
77	{
78	u->next = malloc( sizeof( url_t ) );
79	u = u->next;
80	}
81	else
82	{
83	u->next = axel->url;
84	}
85	}
86	}
87
88	axel->conn[0].conf = axel->conf;
89	if( !conn_set( &axel->conn[0], axel->url->text ) )
90	{
91	axel_message( axel, _("Could not parse URL.\n") );
92	axel->ready = -1;
93	return( axel );
94	}
95
96	axel->conn[0].local_if = axel->conf->interfaces->text;
97	axel->conf->interfaces = axel->conf->interfaces->next;
98
99	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
100	http_decode( axel->filename );
101	if( axel->filename == 0 ) / Index page == no fn */
102	strncpy( axel->filename, axel->conf->default_filename, MAX_STRING );
103	if( ( s = strchr( axel->filename, '?' ) ) != NULL && axel->conf->strip_cgi_parameters )
104	s = 0; / Get rid of CGI parameters */
105
106	if( !conn_init( &axel->conn[0] ) )
107	{
108	axel_message( axel, axel->conn[0].message );
109	axel->ready = -1;
110	return( axel );
111	}
112
113	/* This does more than just checking the file size, it all depends
114	on the protocol used. */
115	if( !conn_info( &axel->conn[0] ) )
116	{
117	axel_message( axel, axel->conn[0].message );
118	axel->ready = -1;
119	return( axel );
120	}
121	s = conn_url( axel->conn );
122	strncpy( axel->url->text, s, MAX_STRING );
123	if( ( axel->size = axel->conn[0].size ) != INT_MAX )
124	{
125	if( axel->conf->verbose > 0 )
126	axel_message( axel, _("File size: %lld bytes"), axel->size );
127	}
128
129	/* Wildcards in URL --> Get complete filename */
130	if( strchr( axel->filename, '*' ) \|\| strchr( axel->filename, '?' ) )
131	strncpy( axel->filename, axel->conn[0].file, MAX_STRING );
132
133	return( axel );
134	}
135
136	/* Open a local file to store the downloaded data */
137	int axel_open( axel_t *axel )
138	{
139	int i, fd;
140	long long int j;
141
142	if( axel->conf->verbose > 0 )
143	axel_message( axel, _("Opening output file %s"), axel->filename );
144	snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
145
146	axel->outfd = -1;
147
148	/* Check whether server knows about RESTart and switch back to
149	single connection download if necessary */
150	if( !axel->conn[0].supported )
151	{
152	axel_message( axel, _("Server unsupported, "
153	"starting from scratch with one connection.") );
154	axel->conf->num_connections = 1;
155	axel->conn = realloc( axel->conn, sizeof( conn_t ) );
156	axel_divide( axel );
157	}
158	else if( ( fd = open( buffer, O_RDONLY ) ) != -1 )
159	{
160	read( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
161
162	axel->conn = realloc( axel->conn, sizeof( conn_t ) * axel->conf->num_connections );
163	memset( axel->conn + 1, 0, sizeof( conn_t ) * ( axel->conf->num_connections - 1 ) );
164
165	axel_divide( axel );
166
167	read( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
168	for( i = 0; i < axel->conf->num_connections; i ++ )
169	read( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
170
171	axel_message( axel, _("State file found: %lld bytes downloaded, %lld to go."),
172	axel->bytes_done, axel->size - axel->bytes_done );
173
174	close( fd );
175
176	if( ( axel->outfd = open( axel->filename, O_WRONLY, 0666 ) ) == -1 )
177	{
178	axel_message( axel, _("Error opening local file") );
179	return( 0 );
180	}
181	}
182
183	/* If outfd == -1 we have to start from scrath now */
184	if( axel->outfd == -1 )
185	{
186	axel_divide( axel );
187
188	if( ( axel->outfd = open( axel->filename, O_CREAT \| O_WRONLY, 0666 ) ) == -1 )
189	{
190	axel_message( axel, _("Error opening local file") );
191	return( 0 );
192	}
193
194	/* And check whether the filesystem can handle seeks to
195	past-EOF areas.. Speeds things up. :) AFAIK this
196	should just not happen: */
197	if( lseek( axel->outfd, axel->size, SEEK_SET ) == -1 && axel->conf->num_connections > 1 )
198	{
199	/* But if the OS/fs does not allow to seek behind
200	EOF, we have to fill the file with zeroes before
201	starting. Slow.. */
202	axel_message( axel, _("Crappy filesystem/OS.. Working around. :-(") );
203	lseek( axel->outfd, 0, SEEK_SET );
204	memset( buffer, 0, axel->conf->buffer_size );
205	j = axel->size;
206	while( j > 0 )
207	{
208	write( axel->outfd, buffer, min( j, axel->conf->buffer_size ) );
209	j -= axel->conf->buffer_size;
210	}
211	}
212	}
213
214	return( 1 );
215	}
216
217	/* Start downloading */
218	void axel_start( axel_t *axel )
219	{
220	int i;
221
222	/* HTTP might've redirected and FTP handles wildcards, so
223	re-scan the URL for every conn */
224	for( i = 0; i < axel->conf->num_connections; i ++ )
225	{
226	conn_set( &axel->conn[i], axel->url->text );
227	axel->url = axel->url->next;
228	axel->conn[i].local_if = axel->conf->interfaces->text;
229	axel->conf->interfaces = axel->conf->interfaces->next;
230	axel->conn[i].conf = axel->conf;
231	if( i ) axel->conn[i].supported = 1;
232	}
233
234	if( axel->conf->verbose > 0 )
235	axel_message( axel, _("Starting download") );
236
237	for( i = 0; i < axel->conf->num_connections; i ++ )
238	if( axel->conn[i].currentbyte <= axel->conn[i].lastbyte )
239	{
240	if( axel->conf->verbose >= 2 )
241	axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
242	i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
243	if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) != 0 )
244	{
245	axel_message( axel, _("pthread error!!!") );
246	axel->ready = -1;
247	}
248	else
249	{
250	axel->conn[i].last_transfer = gettime();
251	axel->conn[i].state = 1;
252	}
253	}
254
255	/* The real downloading will start now, so let's start counting */
256	axel->start_time = gettime();
257	axel->ready = 0;
258	}
259
260	/* Main 'loop' */
261	void axel_do( axel_t *axel )
262	{
263	fd_set fds[1];
264	int hifd, i;
265	long long int remaining,size;
266	struct timeval timeval[1];
267
268	/* Create statefile if necessary */
269	if( gettime() > axel->next_state )
270	{
271	save_state( axel );
272	axel->next_state = gettime() + axel->conf->save_state_interval;
273	}
274
275	/* Wait for data on (one of) the connections */
276	FD_ZERO( fds );
277	hifd = 0;
278	for( i = 0; i < axel->conf->num_connections; i ++ )
279	{
280	if( axel->conn[i].enabled )
281	FD_SET( axel->conn[i].fd, fds );
282	hifd = max( hifd, axel->conn[i].fd );
283	}
284	if( hifd == 0 )
285	{
286	/* No connections yet. Wait... */
287	usleep( 100000 );
288	goto conn_check;
289	}
290	else
291	{
292	timeval->tv_sec = 0;
293	timeval->tv_usec = 100000;
294	/* A select() error probably means it was interrupted
295	by a signal, or that something else's very wrong... */
296	if( select( hifd + 1, fds, NULL, NULL, timeval ) == -1 )
297	{
298	axel->ready = -1;
299	return;
300	}
301	}
302
303	/* Handle connections which need attention */
304	for( i = 0; i < axel->conf->num_connections; i ++ )
305	if( axel->conn[i].enabled ) {
306	if( FD_ISSET( axel->conn[i].fd, fds ) )
307	{
308	axel->conn[i].last_transfer = gettime();
309	size = read( axel->conn[i].fd, buffer, axel->conf->buffer_size );
310	if( size == -1 )
311	{
312	if( axel->conf->verbose )
313	{
314	axel_message( axel, _("Error on connection %i! "
315	"Connection closed"), i );
316	}
317	axel->conn[i].enabled = 0;
318	conn_disconnect( &axel->conn[i] );
319	continue;
320	}
321	else if( size == 0 )
322	{
323	if( axel->conf->verbose )
324	{
325	/* Only abnormal behaviour if: */
326	if( axel->conn[i].currentbyte < axel->conn[i].lastbyte && axel->size != INT_MAX )
327	{
328	axel_message( axel, _("Connection %i unexpectedly closed"), i );
329	}
330	else
331	{
332	axel_message( axel, _("Connection %i finished"), i );
333	}
334	}
335	if( !axel->conn[0].supported )
336	{
337	axel->ready = 1;
338	}
339	axel->conn[i].enabled = 0;
340	conn_disconnect( &axel->conn[i] );
341	continue;
342	}
343	/* remaining == Bytes to go */
344	remaining = axel->conn[i].lastbyte - axel->conn[i].currentbyte + 1;
345	if( remaining < size )
346	{
347	if( axel->conf->verbose )
348	{
349	axel_message( axel, _("Connection %i finished"), i );
350	}
351	axel->conn[i].enabled = 0;
352	conn_disconnect( &axel->conn[i] );
353	size = remaining;
354	/* Don't terminate, still stuff to write! */
355	}
356	/* This should always succeed.. */
357	lseek( axel->outfd, axel->conn[i].currentbyte, SEEK_SET );
358	if( write( axel->outfd, buffer, size ) != size )
359	{
360
361	axel_message( axel, _("Write error!") );
362	axel->ready = -1;
363	return;
364	}
365	axel->conn[i].currentbyte += size;
366	axel->bytes_done += size;
367	}
368	else
369	{
370	if( gettime() > axel->conn[i].last_transfer + axel->conf->connection_timeout )
371	{
372	if( axel->conf->verbose )
373	axel_message( axel, _("Connection %i timed out"), i );
374	conn_disconnect( &axel->conn[i] );
375	axel->conn[i].enabled = 0;
376	}
377	} }
378
379	if( axel->ready )
380	return;
381
382	conn_check:
383	/* Look for aborted connections and attempt to restart them. */
384	for( i = 0; i < axel->conf->num_connections; i ++ )
385	{
386	if( !axel->conn[i].enabled && axel->conn[i].currentbyte < axel->conn[i].lastbyte )
387	{
388	if( axel->conn[i].state == 0 )
389	{
390	// Wait for termination of this thread
391	pthread_join(*(axel->conn[i].setup_thread), NULL);
392
393	conn_set( &axel->conn[i], axel->url->text );
394	axel->url = axel->url->next;
395	/* axel->conn[i].local_if = axel->conf->interfaces->text;
396	axel->conf->interfaces = axel->conf->interfaces->next; */
397	if( axel->conf->verbose >= 2 )
398	axel_message( axel, _("Connection %i downloading from %s:%i using interface %s"),
399	i, axel->conn[i].host, axel->conn[i].port, axel->conn[i].local_if );
400	if( pthread_create( axel->conn[i].setup_thread, NULL, setup_thread, &axel->conn[i] ) == 0 )
401	{
402	axel->conn[i].state = 1;
403	axel->conn[i].last_transfer = gettime();
404	}
405	else
406	{
407	axel_message( axel, _("pthread error!!!") );
408	axel->ready = -1;
409	}
410	}
411	else
412	{
413	if( gettime() > axel->conn[i].last_transfer + axel->conf->reconnect_delay )
414	{
415	pthread_cancel( *axel->conn[i].setup_thread );
416	axel->conn[i].state = 0;
417	}
418	}
419	}
420	}
421
422	/* Calculate current average speed and finish_time */
423	axel->bytes_per_second = (int) ( (double) ( axel->bytes_done - axel->start_byte ) / ( gettime() - axel->start_time ) );
424	axel->finish_time = (int) ( axel->start_time + (double) ( axel->size - axel->start_byte ) / axel->bytes_per_second );
425
426	/* Check speed. If too high, delay for some time to slow things
427	down a bit. I think a 5% deviation should be acceptable. */
428	if( axel->conf->max_speed > 0 )
429	{
430	if( (float) axel->bytes_per_second / axel->conf->max_speed > 1.05 )
431	axel->delay_time += 10000;
432	else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) && ( axel->delay_time >= 10000 ) )
433	axel->delay_time -= 10000;
434	else if( ( (float) axel->bytes_per_second / axel->conf->max_speed < 0.95 ) )
435	axel->delay_time = 0;
436	usleep( axel->delay_time );
437	}
438
439	/* Ready? */
440	if( axel->bytes_done == axel->size )
441	axel->ready = 1;
442	}
443
444	/* Close an axel connection */
445	void axel_close( axel_t *axel )
446	{
447	int i;
448	message_t *m;
449
450	/* Terminate any thread still running */
451	for( i = 0; i < axel->conf->num_connections; i ++ )
452	/* don't try to kill non existing thread */
453	if ( *axel->conn[i].setup_thread != 0 )
454	pthread_cancel( *axel->conn[i].setup_thread );
455
456	/* Delete state file if necessary */
457	if( axel->ready == 1 )
458	{
459	snprintf( buffer, MAX_STRING, "%s.st", axel->filename );
460	unlink( buffer );
461	}
462	/* Else: Create it.. */
463	else if( axel->bytes_done > 0 )
464	{
465	save_state( axel );
466	}
467
468	/* Delete any message not processed yet */
469	while( axel->message )
470	{
471	m = axel->message;
472	axel->message = axel->message->next;
473	free( m );
474	}
475
476	/* Close all connections and local file */
477	close( axel->outfd );
478	for( i = 0; i < axel->conf->num_connections; i ++ )
479	conn_disconnect( &axel->conn[i] );
480
481	free( axel->conn );
482	free( axel );
483	}
484
485	/* time() with more precision */
486	double gettime()
487	{
488	struct timeval time[1];
489
490	gettimeofday( time, 0 );
491	return( (double) time->tv_sec + (double) time->tv_usec / 1000000 );
492	}
493
494	/* Save the state of the current download */
495	void save_state( axel_t *axel )
496	{
497	int fd, i;
498	char fn[MAX_STRING+4];
499
500	/* No use for such a file if the server doesn't support
501	resuming anyway.. */
502	if( !axel->conn[0].supported )
503	return;
504
505	snprintf( fn, MAX_STRING, "%s.st", axel->filename );
506	if( ( fd = open( fn, O_CREAT \| O_TRUNC \| O_WRONLY, 0666 ) ) == -1 )
507	{
508	return; /* Not 100% fatal.. */
509	}
510	write( fd, &axel->conf->num_connections, sizeof( axel->conf->num_connections ) );
511	write( fd, &axel->bytes_done, sizeof( axel->bytes_done ) );
512	for( i = 0; i < axel->conf->num_connections; i ++ )
513	{
514	write( fd, &axel->conn[i].currentbyte, sizeof( axel->conn[i].currentbyte ) );
515	}
516	close( fd );
517	}
518
519	/* Thread used to set up a connection */
520	void setup_thread( void c )
521	{
522	conn_t *conn = c;
523	int oldstate;
524
525	/* Allow this thread to be killed at any time. */
526	pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
527	pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
528
529	if( conn_setup( conn ) )
530	{
531	conn->last_transfer = gettime();
532	if( conn_exec( conn ) )
533	{
534	conn->last_transfer = gettime();
535	conn->enabled = 1;
536	conn->state = 0;
537	return( NULL );
538	}
539	}
540
541	conn_disconnect( conn );
542	conn->state = 0;
543	return( NULL );
544	}
545
546	/* Add a message to the axel->message structure */
547	static void axel_message( axel_t axel, char format, ... )
548	{
549	message_t m = malloc( sizeof( message_t ) ), n = axel->message;
550	va_list params;
551
552	memset( m, 0, sizeof( message_t ) );
553	va_start( params, format );
554	vsnprintf( m->text, MAX_STRING, format, params );
555	va_end( params );
556
557	if( axel->message == NULL )
558	{
559	axel->message = m;
560	}
561	else
562	{
563	while( n->next != NULL )
564	n = n->next;
565	n->next = m;
566	}
567	}
568
569	/* Divide the file and set the locations for each connection */
570	static void axel_divide( axel_t *axel )
571	{
572	int i;
573
574	axel->conn[0].currentbyte = 0;
575	axel->conn[0].lastbyte = axel->size / axel->conf->num_connections - 1;
576	for( i = 1; i < axel->conf->num_connections; i ++ )
577	{
578	#ifdef DEBUG
579	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
580	#endif
581	axel->conn[i].currentbyte = axel->conn[i-1].lastbyte + 1;
582	axel->conn[i].lastbyte = axel->conn[i].currentbyte + axel->size / axel->conf->num_connections;
583	}
584	axel->conn[axel->conf->num_connections-1].lastbyte = axel->size - 1;
585	#ifdef DEBUG
586	printf( "Downloading %lld-%lld using conn. %i\n", axel->conn[i-1].currentbyte, axel->conn[i-1].lastbyte, i - 1 );
587	#endif
588	}