axel/trunk/search.c

  /********************************************************************\
  * Axel -- A lighter download accelerator for Linux and other Unices. *
  *                                                                    *
  * Copyright 2001 Wilmer van der Gaast                                *
  \********************************************************************/

/* filesearching.com searcher                                           */

/*
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License with
  the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
  if not, write to the Free Software Foundation, Inc., 59 Temple Place,
  Suite 330, Boston, MA  02111-1307  USA
*/

#include "axel.h"

static char *strrstr( char *haystack, char *needle );
static void *search_speedtest( void *r );
static int search_sortlist_qsort( const void *a, const void *b );

#ifdef STANDALONE
int main( int argc, char *argv[] )
{
        conf_t conf[1];
        search_t *res;
        int i, j;
        
        if( argc != 2 )
        {
                fprintf( stderr, "Incorrect amount of arguments\n" );
                return( 1 );
        }
        
        conf_init( conf );
        
        res = malloc( sizeof( search_t ) * ( conf->search_amount + 1 ) );
        memset( res, 0, sizeof( search_t ) * ( conf->search_amount + 1 ) );
        res->conf = conf;
        
        i = search_makelist( res, argv[1] );
        if( i == -1 )
        {
                fprintf( stderr, "File not found\n" );
                return( 1 );
        }
        printf( "%i usable mirrors:\n", search_getspeeds( res, i ) );
        search_sortlist( res, i );
        for( j = 0; j < i; j ++ )
                printf( "%-70.70s %5i\n", res[j].url, res[j].speed );
        
        return( 0 );
}
#endif

int search_makelist( search_t *results, char *url )
{
        int i, size = 8192, j = 0;
        char *s, *s1, *s2, *s3;
        conn_t conn[1];
        double t;
        
        memset( conn, 0, sizeof( conn_t ) );
        
        conn->conf = results->conf;
        t = gettime();
        if( !conn_set( conn, url ) )
                return( -1 );
        if( !conn_init( conn ) )
                return( -1 );
        if( !conn_info( conn ) )
                return( -1 );
        
        strcpy( results[0].url, url );
        results[0].speed = 1 + 1000 * ( gettime() - t );
        results[0].size = conn->size;
        
        s = malloc( size );
        
        sprintf( s, "http://www.filesearching.com/cgi-bin/s?q=%s&w=a&l=en&"
                "t=f&e=on&m=%i&o=n&s1=%lld&s2=%lld&x=15&y=15",
                conn->file, results->conf->search_amount,
                conn->size, conn->size );
        
        conn_disconnect( conn );
        memset( conn, 0, sizeof( conn_t ) );
        conn->conf = results->conf;
        
        if( !conn_set( conn, s ) )
        {
                free( s );
                return( 1 );
        }
        if( !conn_setup( conn ) )
        {
                free( s );
                return( 1 );
        }
        if( !conn_exec( conn ) )
        {
                free( s );
                return( 1 );
        }
        
        while( ( i = read( conn->fd, s + j, size - j ) ) > 0 )
        {
                j += i;
                if( j + 10 >= size )
                {
                        size *= 2;
                        s = realloc( s, size );
                        memset( s + size / 2, 0, size / 2 );
                }
        }

        conn_disconnect( conn );
        
        s1 = strstr( s, "<pre class=list" );
        s1 = strchr( s1, '\n' ) + 1;
        if( strstr( s1, "</pre>" ) == NULL )
        {
                /* Incomplete list                                      */
                free( s );
                return( 1 );
        }
        for( i = 1; strncmp( s1, "</pre>", 6 ) && i < results->conf->search_amount && *s1; i ++ )
        {
                s3 = strchr( s1, '\n' ); *s3 = 0;
                s2 = strrstr( s1, "<a href=" ) + 8;
                *s3 = '\n';
                s3 = strchr( s2, ' ' ); *s3 = 0;
                if( strcmp( results[0].url, s2 ) )
                {
                        strncpy( results[i].url, s2, MAX_STRING );
                        results[i].size = results[0].size;
                        results[i].conf = results->conf;
                }
                else
                {
                        /* The original URL might show up               */
                        i --;
                }
                for( s1 = s3; *s1 != '\n'; s1 ++ );
                s1 ++;
        }
        
        free( s );
        
        return( i );
}

#define SPEED_ACTIVE    -1
#define SPEED_ERROR     -2
#define SPEED_DONE      -3      /* Or >0                                */

int search_getspeeds( search_t *results, int count )
{
        int i, running = 0, done = 0, correct = 0;
        
        for( i = 0; i < count; i ++ ) if( results[i].speed )
        {
                results[i].speed_start_time = 0;
                done ++;
                if( results[i].speed > 0 )
                        correct ++;
        }
        
        while( done < count )
        {
                for( i = 0; i < count; i ++ )
                {
                        if( running < results->conf->search_threads && !results[i].speed )
                        {
                                results[i].speed = SPEED_ACTIVE;
                                results[i].speed_start_time = gettime();
                                if( pthread_create( results[i].speed_thread,
                                        NULL, search_speedtest, &results[i] ) == 0 )
                                {
                                        running ++;
                                        break;
                                }
                                else
                                {
                                        return( 0 );
                                }
                        }
                        else if( ( results[i].speed == SPEED_ACTIVE ) &&
                                ( gettime() > results[i].speed_start_time + results->conf->search_timeout ) )
                        {
                                pthread_cancel( *results[i].speed_thread );
                                results[i].speed = SPEED_DONE;
                                running --;
                                done ++;
                                break;
                        }
                        else if( results[i].speed > 0 && results[i].speed_start_time )
                        {
                                results[i].speed_start_time = 0;
                                running --;
                                correct ++;
                                done ++;
                                break;
                        }
                        else if( results[i].speed == SPEED_ERROR )
                        {
                                results[i].speed = SPEED_DONE;
                                running --;
                                done ++;
                                break;
                        }
                }
                if( i == count )
                {
                        usleep( 100000 );
                }
        }

        return( correct );
}

void *search_speedtest( void *r )
{
        search_t *results = r;
        conn_t conn[1];
        int oldstate;
        
        /* Allow this thread to be killed at any time.                  */
        pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
        pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
        
        memset( conn, 0, sizeof( conn_t ) );
        conn->conf = results->conf;
        if( !conn_set( conn, results->url ) )
                results->speed = SPEED_ERROR;
        else if( !conn_init( conn ) )
                results->speed = SPEED_ERROR;
        else if( !conn_info( conn ) )
                results->speed = SPEED_ERROR;
        else if( conn->size == results->size )
                /* Add one because it mustn't be zero                   */
                results->speed = 1 + 1000 * ( gettime() - results->speed_start_time );
        else
                results->speed = SPEED_ERROR;

        conn_disconnect( conn );
        
        return( NULL );
}

char *strrstr( char *haystack, char *needle )
{
        int i, j;
        
        for( i = strlen( haystack ) - strlen( needle ); i > 0; i -- )
        {
                for( j = 0; needle[j] && haystack[i+j] == needle[j]; j ++ );
                if( !needle[j] )
                        return( haystack + i );
        }
        
        return( NULL );
}

void search_sortlist( search_t *results, int count )
{
        qsort( results, count, sizeof( search_t ), search_sortlist_qsort );
}

int search_sortlist_qsort( const void *a, const void *b )
{
        if( ((search_t *)a)->speed < 0 && ((search_t *)b)->speed > 0 )
                return( 1 );
        if( ((search_t *)a)->speed > 0 && ((search_t *)b)->speed < 0 )
                return( -1 );
        if( ((search_t *)a)->speed < ((search_t *)b)->speed )
                return( -1 );
        else
                return( ((search_t *)a)->speed > ((search_t *)b)->speed );
}
1	/********************************************************************\
2	* Axel -- A lighter download accelerator for Linux and other Unices. *
3	* *
4	* Copyright 2001 Wilmer van der Gaast *
5	\********************************************************************/
6
7	/* filesearching.com searcher */
8
9	/*
10	This program is free software; you can redistribute it and/or modify
11	it under the terms of the GNU General Public License as published by
12	the Free Software Foundation; either version 2 of the License, or
13	(at your option) any later version.
14
15	This program is distributed in the hope that it will be useful,
16	but WITHOUT ANY WARRANTY; without even the implied warranty of
17	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	GNU General Public License for more details.
19
20	You should have received a copy of the GNU General Public License with
21	the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
22	if not, write to the Free Software Foundation, Inc., 59 Temple Place,
23	Suite 330, Boston, MA 02111-1307 USA
24	*/
25
26	#include "axel.h"
27
28	static char strrstr( char haystack, char *needle );
29	static void search_speedtest( void r );
30	static int search_sortlist_qsort( const void a, const void b );
31
32	#ifdef STANDALONE
33	int main( int argc, char *argv[] )
34	{
35	conf_t conf[1];
36	search_t *res;
37	int i, j;
38
39	if( argc != 2 )
40	{
41	fprintf( stderr, "Incorrect amount of arguments\n" );
42	return( 1 );
43	}
44
45	conf_init( conf );
46
47	res = malloc( sizeof( search_t ) * ( conf->search_amount + 1 ) );
48	memset( res, 0, sizeof( search_t ) * ( conf->search_amount + 1 ) );
49	res->conf = conf;
50
51	i = search_makelist( res, argv[1] );
52	if( i == -1 )
53	{
54	fprintf( stderr, "File not found\n" );
55	return( 1 );
56	}
57	printf( "%i usable mirrors:\n", search_getspeeds( res, i ) );
58	search_sortlist( res, i );
59	for( j = 0; j < i; j ++ )
60	printf( "%-70.70s %5i\n", res[j].url, res[j].speed );
61
62	return( 0 );
63	}
64	#endif
65
66	int search_makelist( search_t results, char url )
67	{
68	int i, size = 8192, j = 0;
69	char s, s1, s2, s3;
70	conn_t conn[1];
71	double t;
72
73	memset( conn, 0, sizeof( conn_t ) );
74
75	conn->conf = results->conf;
76	t = gettime();
77	if( !conn_set( conn, url ) )
78	return( -1 );
79	if( !conn_init( conn ) )
80	return( -1 );
81	if( !conn_info( conn ) )
82	return( -1 );
83
84	strcpy( results[0].url, url );
85	results[0].speed = 1 + 1000 * ( gettime() - t );
86	results[0].size = conn->size;
87
88	s = malloc( size );
89
90	sprintf( s, "http://www.filesearching.com/cgi-bin/s?q=%s&w=a&l=en&"
91	"t=f&e=on&m=%i&o=n&s1=%lld&s2=%lld&x=15&y=15",
92	conn->file, results->conf->search_amount,
93	conn->size, conn->size );
94
95	conn_disconnect( conn );
96	memset( conn, 0, sizeof( conn_t ) );
97	conn->conf = results->conf;
98
99	if( !conn_set( conn, s ) )
100	{
101	free( s );
102	return( 1 );
103	}
104	if( !conn_setup( conn ) )
105	{
106	free( s );
107	return( 1 );
108	}
109	if( !conn_exec( conn ) )
110	{
111	free( s );
112	return( 1 );
113	}
114
115	while( ( i = read( conn->fd, s + j, size - j ) ) > 0 )
116	{
117	j += i;
118	if( j + 10 >= size )
119	{
120	size *= 2;
121	s = realloc( s, size );
122	memset( s + size / 2, 0, size / 2 );
123	}
124	}
125
126	conn_disconnect( conn );
127
128	s1 = strstr( s, "<pre class=list" );
129	s1 = strchr( s1, '\n' ) + 1;
130	if( strstr( s1, "</pre>" ) == NULL )
131	{
132	/* Incomplete list */
133	free( s );
134	return( 1 );
135	}
136	for( i = 1; strncmp( s1, "</pre>", 6 ) && i < results->conf->search_amount && *s1; i ++ )
137	{
138	s3 = strchr( s1, '\n' ); *s3 = 0;
139	s2 = strrstr( s1, "<a href=" ) + 8;
140	*s3 = '\n';
141	s3 = strchr( s2, ' ' ); *s3 = 0;
142	if( strcmp( results[0].url, s2 ) )
143	{
144	strncpy( results[i].url, s2, MAX_STRING );
145	results[i].size = results[0].size;
146	results[i].conf = results->conf;
147	}
148	else
149	{
150	/* The original URL might show up */
151	i --;
152	}
153	for( s1 = s3; *s1 != '\n'; s1 ++ );
154	s1 ++;
155	}
156
157	free( s );
158
159	return( i );
160	}
161
162	#define SPEED_ACTIVE -1
163	#define SPEED_ERROR -2
164	#define SPEED_DONE -3 /* Or >0 */
165
166	int search_getspeeds( search_t *results, int count )
167	{
168	int i, running = 0, done = 0, correct = 0;
169
170	for( i = 0; i < count; i ++ ) if( results[i].speed )
171	{
172	results[i].speed_start_time = 0;
173	done ++;
174	if( results[i].speed > 0 )
175	correct ++;
176	}
177
178	while( done < count )
179	{
180	for( i = 0; i < count; i ++ )
181	{
182	if( running < results->conf->search_threads && !results[i].speed )
183	{
184	results[i].speed = SPEED_ACTIVE;
185	results[i].speed_start_time = gettime();
186	if( pthread_create( results[i].speed_thread,
187	NULL, search_speedtest, &results[i] ) == 0 )
188	{
189	running ++;
190	break;
191	}
192	else
193	{
194	return( 0 );
195	}
196	}
197	else if( ( results[i].speed == SPEED_ACTIVE ) &&
198	( gettime() > results[i].speed_start_time + results->conf->search_timeout ) )
199	{
200	pthread_cancel( *results[i].speed_thread );
201	results[i].speed = SPEED_DONE;
202	running --;
203	done ++;
204	break;
205	}
206	else if( results[i].speed > 0 && results[i].speed_start_time )
207	{
208	results[i].speed_start_time = 0;
209	running --;
210	correct ++;
211	done ++;
212	break;
213	}
214	else if( results[i].speed == SPEED_ERROR )
215	{
216	results[i].speed = SPEED_DONE;
217	running --;
218	done ++;
219	break;
220	}
221	}
222	if( i == count )
223	{
224	usleep( 100000 );
225	}
226	}
227
228	return( correct );
229	}
230
231	void search_speedtest( void r )
232	{
233	search_t *results = r;
234	conn_t conn[1];
235	int oldstate;
236
237	/* Allow this thread to be killed at any time. */
238	pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, &oldstate );
239	pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate );
240
241	memset( conn, 0, sizeof( conn_t ) );
242	conn->conf = results->conf;
243	if( !conn_set( conn, results->url ) )
244	results->speed = SPEED_ERROR;
245	else if( !conn_init( conn ) )
246	results->speed = SPEED_ERROR;
247	else if( !conn_info( conn ) )
248	results->speed = SPEED_ERROR;
249	else if( conn->size == results->size )
250	/* Add one because it mustn't be zero */
251	results->speed = 1 + 1000 * ( gettime() - results->speed_start_time );
252	else
253	results->speed = SPEED_ERROR;
254
255	conn_disconnect( conn );
256
257	return( NULL );
258	}
259
260	char strrstr( char haystack, char *needle )
261	{
262	int i, j;
263
264	for( i = strlen( haystack ) - strlen( needle ); i > 0; i -- )
265	{
266	for( j = 0; needle[j] && haystack[i+j] == needle[j]; j ++ );
267	if( !needle[j] )
268	return( haystack + i );
269	}
270
271	return( NULL );
272	}
273
274	void search_sortlist( search_t *results, int count )
275	{
276	qsort( results, count, sizeof( search_t ), search_sortlist_qsort );
277	}
278
279	int search_sortlist_qsort( const void a, const void b )
280	{
281	if( ((search_t )a)->speed < 0 && ((search_t )b)->speed > 0 )
282	return( 1 );
283	if( ((search_t )a)->speed > 0 && ((search_t )b)->speed < 0 )
284	return( -1 );
285	if( ((search_t )a)->speed < ((search_t )b)->speed )
286	return( -1 );
287	else
288	return( ((search_t )a)->speed > ((search_t )b)->speed );
289	}