summaryrefslogtreecommitdiff
path: root/segget
diff options
context:
space:
mode:
Diffstat (limited to 'segget')
-rw-r--r--segget/Makefile3
-rw-r--r--segget/connection.cpp32
-rw-r--r--segget/connection.h2
-rw-r--r--segget/distfile.cpp109
-rw-r--r--segget/distfile.h3
-rw-r--r--segget/mirror.cpp2
-rw-r--r--segget/network0_mirrors.conf5
-rw-r--r--segget/network1_mirrors.conf2
-rw-r--r--segget/networkbroker.cpp5
-rw-r--r--segget/networkbroker.h10
-rw-r--r--segget/phase.cpp37
-rw-r--r--segget/phase.h43
-rw-r--r--segget/pkg.cpp3
-rw-r--r--segget/response.cpp65
-rw-r--r--segget/response.h61
-rw-r--r--segget/segget.conf6
-rw-r--r--segget/segget.cpp29
-rw-r--r--segget/segment.cpp39
-rw-r--r--segget/segment.h3
19 files changed, 342 insertions, 117 deletions
diff --git a/segget/Makefile b/segget/Makefile
index fe15e2c..ed965bf 100644
--- a/segget/Makefile
+++ b/segget/Makefile
@@ -18,8 +18,7 @@ all: clean $(BINS)
%.o: %.cxx
$(CXX) -c -o $@ $(CXXFLAGS) $^
-segget: segget.o connection.o checksum.o config.o distfile.o mirror.o network.o networkbroker.o pkg.o segment.o \
- settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o
+segget: segget.o connection.o checksum.o config.o distfile.o mirror.o network.o networkbroker.o phase.o pkg.o response.o segment.o settings.o stats.o str.o tui.o utils.o ui_server.o proxyfetcher.o
$(CXX) -o $@ -lncurses $(LIBS) $(CIBS) $^
#%: %.o
# $(CXX) -o $@ $(LIBS) $(CIBS) $^
diff --git a/segget/connection.cpp b/segget/connection.cpp
index d2b0e2c..108c991 100644
--- a/segget/connection.cpp
+++ b/segget/connection.cpp
@@ -28,6 +28,7 @@
uint Tconnection::total_connections=0;
Tconnection connection_array[MAX_CONNECTS];
time_t prev_time;
+
void init_connections(){
for (ulong connection_num=0; connection_num<MAX_CONNECTS; connection_num++){
connection_array[connection_num].connection_num=connection_num;
@@ -47,25 +48,44 @@ void Tconnection::start(CURLM *cm, uint network_number, uint distfile_num, Tsegm
active=true;
debug("Connecting network"+toString(network_num));
- if (network_array[network_number].network_mode==MODE_PROXY_FETCHER){
- connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_number].phase;
+ if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){
+ connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_num].phase;
+ }
+ string url;
+ if (network_array[network_num].network_mode!=MODE_REMOTE){
+ url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+segment->parent_distfile->name;
+ debug(" URL:"+url);
+ }else{
+ url=segment->parent_distfile->url_list[mirror_num];
}
+
+ network_array[network_num].benchmarked_mirror_list[best_mirror_num].start();
network_array[network_num].connect();
- segment->prepare_for_connection(cm, connection_num, network_num, distfile_num, mirror_num);
+ segment->prepare_for_connection(cm, connection_num, network_num, distfile_num, url);
debug("Started connection for distfile: "+segment->parent_distfile->name);
}catch(...){
error_log("Error in connection.cpp: start()");
}
}
-void Tconnection::stop(uint connection_result){
+void Tconnection::stop(int connection_result){
try{
- debug("Finished connection for distfile"+segment->parent_distfile->name+" Status"+toString(connection_result));
+ debug("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result));
+ error_log("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result));
+
msg_clean_connection(connection_num);
active=false;
network_array[network_num].disconnect();
+// network_array[network_num].benchmarked_mirror_list[mirror_num].stop();
segment->segment_file.close();
+ if (connection_result==0){
+ if (! segment->segment_verification_is_ok()){
+ debug("curl_lies - there is a problem downloading segment");
+ error_log("curl_lies - there is a problem downloading segment");
+ connection_result=100;
+ }
+ }
Tdistfile* prnt_distfile=segment->parent_distfile;
prnt_distfile->active_connections_num--;
@@ -124,7 +144,7 @@ void Tconnection::stop(uint connection_result){
// no error => count this one and start new
log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num));
debug(" Successful download "+segment->url);
- Pcurr_mirror=find_mirror(strip_mirror_name(segment->url));
+// already done earlier in this function Pcurr_mirror=find_mirror(strip_mirror_name(segment->url));
Pcurr_mirror->stop(time_left_from(connection_array[connection_num].start_time),segment->segment_size);
segment->status=SDOWNLOADED;
prnt_distfile->inc_dld_segments_count(segment);
diff --git a/segget/connection.h b/segget/connection.h
index d20ad01..03d502f 100644
--- a/segget/connection.h
+++ b/segget/connection.h
@@ -60,7 +60,7 @@ class Tconnection{
start_time(),
segment(0){};
void start(CURLM *cm, uint network_number, uint distfile_num, Tsegment *started_segment, uint best_mirror_num);
- void stop(uint connection_result);
+ void stop(int connection_result);
void inc_bytes_per_last_interval(ulong new_bytes_count);
void show_connection_progress(ulong time_diff);
};
diff --git a/segget/distfile.cpp b/segget/distfile.cpp
index 16d98f0..b27a52f 100644
--- a/segget/distfile.cpp
+++ b/segget/distfile.cpp
@@ -29,8 +29,6 @@
//Make the necessary includes and set up the variables:
using namespace std;
-int Tdistfile::decode_server_response(string server_response){
-
/*
#define R_LM_WAIT_FOR_LOCAL_MIRRORS 100
@@ -56,39 +54,6 @@ int Tdistfile::decode_server_response(string server_response){
#define ALLOW_LOWER_PRIORITY_NETWORKS 205
*/
-int int_server_response=atoi(server_response.c_str());
- switch (int_server_response){
- case R_PF_REJECTED:{
- debug("Server response:"+server_response+" - R_PF_REJECTED");
- return int_server_response;
- };
- case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{
- debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE");
- return int_server_response;
- };
- case R_PF_ADDED_TO_PROXY_QUEUE:{
- debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE");
- return int_server_response;
- };
- case R_PF_ALREADY_WAS_IN_QUEUE:{
- debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE");
- return int_server_response;
- };
- case R_PF_DOWNLOADED:{
- debug("Server response:"+server_response+" - R_PF_DOWNLOADED");
- return int_server_response;
- };
- case R_PF_FAILED:{
- debug("Server response:"+server_response+" - R_PF_FAILED");
- return int_server_response;
- };
- default :{
- debug("Server response:"+server_response+" - unknown => R_PF_FAILED");
- return R_PF_FAILED;
- };
- }
-}
-
int Tdistfile::request(ulong network_num, string msg){
gettimeofday(&network_distfile_brokers_array[network_num].last_request_time, NULL);
int sockfd;
@@ -326,7 +291,9 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne
long best_mirror_num=-1; // the best isn't set let's find it
ulong best_mirror_self_rating=-1;
ulong curr_mirror_self_rating;
+ debug("Choosing mirror for network"+toString(network_num));
for (ulong mirror_num=0; mirror_num<network_array[network_num].benchmarked_mirror_list.size(); mirror_num++){
+ debug("Evaluating mirror:"+network_array[network_num].benchmarked_mirror_list[mirror_num].url);
if (network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num()<settings.max_connections_num_per_mirror){
curr_mirror_self_rating=network_array[network_num].benchmarked_mirror_list[mirror_num].mirror_on_the_wall();
if (curr_mirror_self_rating<best_mirror_self_rating){
@@ -336,16 +303,23 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne
if (best_mirror_self_rating==0)
// 0 can not be improved - it's one of the best
break;
+ }else{
+ debug("Mirror already has:"
+ +toString(network_array[network_num].benchmarked_mirror_list[mirror_num].get_active_num())
+ +" connections which doesn't meet limit:"+toString(settings.max_connections_num_per_mirror));
}
}
if (best_mirror_num!=-1){
debug("Downloading from BEST_LOCAL_MIRROR:"+network_array[network_num].benchmarked_mirror_list[best_mirror_num].url);
- network_array[network_num].benchmarked_mirror_list[best_mirror_num].start();
+
+
+
active_connections_num++;
connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num);
return R_R_DOWNLOAD_STARTED;
}
else{
+ debug("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name);
error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name);
return 1;
}
@@ -372,8 +346,11 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n
if (network_array[network_num].priority==network_priority){
debug(" network_priority="+toString(network_array[network_num].priority));
if (network_array[network_num].network_mode==MODE_LOCAL){
+ debug(" network#:"+toString(network_num)+" has MODE_LOCAL");
+ debug(" phase#"+phaseToString(network_distfile_brokers_array[network_num].phase));
if (network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS){
if (network_array[network_num].has_free_connections()){
+ debug(" has free connections");
// debug(" Allowed network#:"+toString(network_num));
if ((best_local_network_num==-1)
or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){
@@ -381,6 +358,7 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n
debug(" Replace best LOCAL network to network#:"+toString(network_num));
}
}else{
+ debug(" does NOT have free connections");
if (network_array[network_num].only_local_when_possible){
allow_proxy_fetcher_mirrors=false;
debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed");
@@ -388,12 +366,15 @@ uint Tdistfile::provide_local_network(CURLM* cm, uint connection_num, uint seg_n
}
}
}
- if (network_array[network_num].network_mode==MODE_PROXY_FETCHER)
- {
+ if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){
+ debug(" network#:"+toString(network_num)+" has MODE_PROXY_FETCHER");
+ debug(" with phase:"+phaseToString(network_distfile_brokers_array[network_num].phase));
if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS)
or (network_distfile_brokers_array[network_num].phase==E_PROXY_FETCHER_DOWNLOADED))
{
+ debug("phase fits");
if (network_array[network_num].has_free_connections()){
+ debug(" has free connections");
// debug(" Allowed network#:"+toString(network_num));
if ((best_local_network_num==-1)
or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){
@@ -442,23 +423,32 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){
if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){
//replace this one by does_not_reject_connections
// if (network_array[network_num].has_free_connections()){
- if ((best_proxy_fetcher_network_num==-1)
-
-// or
+ if ((network_distfile_brokers_array[network_num].phase==E_USE_AS_LOCAL_MIRRORS)
+ or (network_distfile_brokers_array[network_num].phase==E_ALL_LOCAL_MIRRORS_FAILED)){
+ debug("------//////////// Good phase !!!!!!!!!!!!!!!");
+ decode_server_response(toString(network_distfile_brokers_array[network_num].proxy_fetcher_response));
+ switch (network_distfile_brokers_array[network_num].proxy_fetcher_response){
+ case R_PF_DOWNLOADED:break;
+ case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break;
+ case R_PF_FAILED:break;
+ case R_PF_REJECTED:break;
+ default:{
+ //if less then 30 secs left don't bother proxy-fetcher
+ if (2000>time_left_from(network_distfile_brokers_array[network_num].last_request_time)){
+ debug("====================== Be more patient with proxy-fetcher - network#:"+toString(network_num)
+ +" because time left:"+toString(time_left_from(network_distfile_brokers_array[network_num].last_request_time)));
+ return R_PF_BE_MORE_PATIENT;
+ }else{
// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num)
-
- ){
- switch (network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response){
- case R_PF_DOWNLOADED:break;
- case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:break;
- case R_PF_FAILED:break;
- case R_PF_REJECTED:break;
- default:{
best_proxy_fetcher_network_num=network_num;
+ debug("-------------------------------");
debug(" Replace best_proxy_fetcher network_num to network#:"+toString(best_proxy_fetcher_network_num));
- break;
- }
+ };
+ break;
}
+ }
+ }
+ if (best_proxy_fetcher_network_num!=-1) break;
/*
}else{
if (network_array[network_num].only_local_when_possible){
@@ -469,15 +459,10 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){
}
}
*/
- }
}
}
}
if (best_proxy_fetcher_network_num != -1){
- //if less then 30 secs left don't bother proxy-fetcher
- if (30000>time_left_from(network_distfile_brokers_array[best_proxy_fetcher_network_num].last_request_time)){
- return R_PF_BE_MORE_PATIENT;
- }else{
// request from proxy fethcer
network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response=request(best_proxy_fetcher_network_num, json_data);
if ((network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response==R_PF_DOWNLOADED)
@@ -494,7 +479,6 @@ uint Tdistfile::request_proxy_fetcher_network(uint network_priority){
}
//return - don't switch to low priority networks
return network_distfile_brokers_array[best_proxy_fetcher_network_num].proxy_fetcher_response;
- }
}else{
if (allow_remote_mirrors){
debug(" So best_proxy_fetcher_failed num="+toString(best_proxy_fetcher_network_num));
@@ -555,10 +539,11 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
try{
int result=R_NOT_SET;
for (uint network_priority=10; network_priority>0; network_priority--){
- {
+ do {
+ debug("Switching to local networks with priority:"+toString(network_priority));
result=provide_local_network(cm, connection_num, seg_num, network_priority);
if (result==ALLOW_REQUESTS_TO_PROXY_FETCHERS){
- debug("Switching to proxy-fetcher networks with priority:"+toString(network_priority));
+ debug("Requesting proxy-fetcher networks with priority:"+toString(network_priority));
result=request_proxy_fetcher_network(network_priority);
switch (result){
case ALLOW_REMOTE_NETWORKS:{
@@ -572,7 +557,10 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
}
// don't switch to remote and lower priority networks => return results
case DO_NOT_ALLOW_REMOTE_NETWORKS: return result;
- case R_PF_DOWNLOADED: break;
+ case R_PF_DOWNLOADED: {debug("Yep, R_PF_DONWLOADED => LOOP AGAIN"); break;};
+ // if already in queue, etc...
+ // don't switch to remote and lower priority networks => return results
+ default: return result;
}
}else{
// don't switch to proxy-fetcher requests, remote,
@@ -581,6 +569,7 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
}
// try to download from local mirrors again because one of the proxy-fetcher
// claims that distfile has been donwloaded
+ debug("----- JUST BEFORE THE LOOP ----");
}while (result==R_PF_DOWNLOADED);
}
// haven't found anything suitable
diff --git a/segget/distfile.h b/segget/distfile.h
index ea92fe3..8d42894 100644
--- a/segget/distfile.h
+++ b/segget/distfile.h
@@ -52,7 +52,7 @@
#include "network.h"
#include "networkbroker.h"
#include "segment.h"
-#include "responses.h"
+#include "response.h"
class Tsegment;
//#include <sys/types.h>
@@ -136,7 +136,6 @@ class Tdistfile{
Tdistfile(const Tdistfile &L); // copy constructor
Tdistfile & operator=(const Tdistfile &L);
~Tdistfile();
- int decode_server_response(string server_response);
int request(ulong network_num, string msg);
void init();
bool allows_new_actions();
diff --git a/segget/mirror.cpp b/segget/mirror.cpp
index defec75..b23bd5f 100644
--- a/segget/mirror.cpp
+++ b/segget/mirror.cpp
@@ -54,7 +54,7 @@ void Tmirror::stop(ulong time, uint size){
dld_time+=time/1000;
dld_size+=size;
honesty=1;
- debug(toString(time)+"---"+toString(size));
+ debug("Connnection to Mirror stopped: Time spent:"+toString(time)+"--- Size downloaded:"+toString(size));
active_num--;
}catch(...){
error_log("Error in mirror.cpp: stop()");
diff --git a/segget/network0_mirrors.conf b/segget/network0_mirrors.conf
index 4164689..529a268 100644
--- a/segget/network0_mirrors.conf
+++ b/segget/network0_mirrors.conf
@@ -1,4 +1 @@
-ftp://192.168.56.11/
-ftp://192.168.56.12/
-ftp://192.168.56.13/
-ftp://192.168.56.14/ \ No newline at end of file
+ftp://192.168.56.101/ \ No newline at end of file
diff --git a/segget/network1_mirrors.conf b/segget/network1_mirrors.conf
index 7628506..9406219 100644
--- a/segget/network1_mirrors.conf
+++ b/segget/network1_mirrors.conf
@@ -1 +1 @@
-http://192.168.0.1/ \ No newline at end of file
+http://127.0.0.1/ \ No newline at end of file
diff --git a/segget/networkbroker.cpp b/segget/networkbroker.cpp
index 8062ab6..760bcfe 100644
--- a/segget/networkbroker.cpp
+++ b/segget/networkbroker.cpp
@@ -25,6 +25,11 @@
*/
#include "networkbroker.h"
+
+string phaseToString(int phase_num);
+
+
+
void Tnetwork_distfile_broker::init(ulong network_number){
try{
network_num=network_number;
diff --git a/segget/networkbroker.h b/segget/networkbroker.h
index bb6ba49..602cdb0 100644
--- a/segget/networkbroker.h
+++ b/segget/networkbroker.h
@@ -29,16 +29,10 @@
#include <vector>
#include "shorttypes.h"
#include "network.h"
-#include "responses.h"
+#include "response.h"
+#include "phase.h"
using namespace std;
-enum Tnetwork_distfile_broker_phases{
- E_USE_AS_LOCAL_MIRRORS,
- E_ALL_LOCAL_MIRRORS_FAILED,
- E_PROXY_FETCHER_DOWNLOADED,
- E_ALL_PROXY_FETCHER_MIRRORS_FAILED
-};
-
class Tnetwork_distfile_broker{
public:
// map<string,Tmirror *> benchmarked_mirror_list;
diff --git a/segget/phase.cpp b/segget/phase.cpp
new file mode 100644
index 0000000..04aef35
--- /dev/null
+++ b/segget/phase.cpp
@@ -0,0 +1,37 @@
+/*
+* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>.
+*
+* Project: IDFetch.
+* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine).
+* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead).
+* Mentoring organization: Gentoo Linux.
+* Sponsored by GSOC 2010.
+*
+* This file is part of Segget.
+*
+* Segget is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* Segget is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with Segget; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#include "phase.h"
+
+string phaseToString(int phase_num){
+ switch(phase_num){
+ case E_USE_AS_LOCAL_MIRRORS: return "E_USE_AS_LOCAL_MIRRORS";
+ case E_ALL_LOCAL_MIRRORS_FAILED: return "E_ALL_LOCAL_MIRRORS_FAILED";
+ case E_PROXY_FETCHER_DOWNLOADED: return "E_PROXY_FETCHER_DOWNLOADED";
+ case E_ALL_PROXY_FETCHER_MIRRORS_FAILED: return "E_ALL_PROXY_FETCHER_MIRRORS_FAILED";
+ default: return "Unknown phase";
+ }
+}
diff --git a/segget/phase.h b/segget/phase.h
new file mode 100644
index 0000000..221c33d
--- /dev/null
+++ b/segget/phase.h
@@ -0,0 +1,43 @@
+/*
+* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>.
+*
+* Project: IDFetch.
+* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine).
+* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead).
+* Mentoring organization: Gentoo Linux.
+* Sponsored by GSOC 2010.
+*
+* This file is part of Segget.
+*
+* Segget is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* Segget is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with Segget; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#ifndef __PHASE_H__
+#define __PHASE_H__
+
+#include <string>
+
+using namespace std;
+
+enum Tnetwork_distfile_broker_phases{
+ E_USE_AS_LOCAL_MIRRORS,
+ E_ALL_LOCAL_MIRRORS_FAILED,
+ E_PROXY_FETCHER_DOWNLOADED,
+ E_ALL_PROXY_FETCHER_MIRRORS_FAILED
+};
+
+string phaseToString(int phase_num);
+
+#endif \ No newline at end of file
diff --git a/segget/pkg.cpp b/segget/pkg.cpp
index 72cf539..3a02f74 100644
--- a/segget/pkg.cpp
+++ b/segget/pkg.cpp
@@ -56,7 +56,8 @@ int Tpkg::try_adding_distfile_to_proxy_fetchers_queue(json_object* json_obj_dist
string distfile_name;
try{
distfile_name=json_object_get_string(json_object_object_get(json_obj_distfile,"name"));
- if (is_symlink_restricted(distfile_name)){
+ if (is_symlink_restricted(distfile_name)!=-1){
+ debug("PROXY_FETCHER: distfile: "+distfile_name+" restricted (name matches restriting pattern");
return R_PF_REJECTED;
}
for (ulong distfile_num=0; distfile_num<distfile_count; distfile_num++){
diff --git a/segget/response.cpp b/segget/response.cpp
new file mode 100644
index 0000000..5db952e
--- /dev/null
+++ b/segget/response.cpp
@@ -0,0 +1,65 @@
+/*
+* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>.
+*
+* Project: IDFetch.
+* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine).
+* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead).
+* Mentoring organization: Gentoo Linux.
+* Sponsored by GSOC 2010.
+*
+* This file is part of Segget.
+*
+* Segget is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* Segget is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with Segget; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#include "response.h"
+int decode_server_response(string server_response){
+ int int_server_response=atoi(server_response.c_str());
+ switch (int_server_response){
+ case R_PF_NOT_REQUESTED_YET:{
+ debug("Server response:"+server_response+" - R_PF_NOT_REQUESTED_YET");
+ return int_server_response;
+ };
+ case R_PF_REJECTED:{
+ debug("Server response:"+server_response+" - R_PF_REJECTED");
+ return int_server_response;
+ };
+ case R_PF_ERROR_ADDING_TO_PROXY_QUEUE:{
+ debug("Server response:"+server_response+" - R_PF_ERROR_ADDING_TO_PROXY_QUEUE");
+ return int_server_response;
+ };
+ case R_PF_ADDED_TO_PROXY_QUEUE:{
+ debug("Server response:"+server_response+" - R_PF_ADDED_TO_PROXY_QUEUE");
+ return int_server_response;
+ };
+ case R_PF_ALREADY_WAS_IN_QUEUE:{
+ debug("Server response:"+server_response+" - R_PF_ALREADY_WAS_IN_QUEUE");
+ return int_server_response;
+ };
+ case R_PF_DOWNLOADED:{
+ debug("Server response:"+server_response+" - R_PF_DOWNLOADED");
+ return int_server_response;
+ };
+ case R_PF_FAILED:{
+ debug("Server response:"+server_response+" - R_PF_FAILED");
+ return int_server_response;
+ };
+ default :{
+ debug("Server response:"+server_response+" - unknown => R_PF_FAILED");
+ return R_PF_FAILED;
+ };
+ }
+}
+
diff --git a/segget/response.h b/segget/response.h
new file mode 100644
index 0000000..77ec9fe
--- /dev/null
+++ b/segget/response.h
@@ -0,0 +1,61 @@
+/*
+* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>.
+*
+* Project: IDFetch.
+* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine).
+* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead).
+* Mentoring organization: Gentoo Linux.
+* Sponsored by GSOC 2010.
+*
+* This file is part of Segget.
+*
+* Segget is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* Segget is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with Segget; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#ifndef __RESPONSE_H__
+#define __RESPONSE_H__
+
+#include "tui.h"
+
+using namespace std;
+
+#define R_NOT_SET 300
+#define R_LM_WAIT_FOR_LOCAL_MIRRORS 50
+
+#define R_PF_NOT_REQUESTED_YET 100
+#define R_PF_ADDED_TO_PROXY_QUEUE 101
+#define R_PF_ALREADY_WAS_IN_QUEUE 102
+#define R_PF_DOWNLOADED 103
+#define R_PF_BE_MORE_PATIENT 104
+#define R_PF_ERROR_ADDING_TO_PROXY_QUEUE 105
+#define R_PF_FAILED 106
+#define R_PF_REJECTED 107
+
+// 0 for succesfull return of provide_segment()
+#define R_R_DOWNLOAD_STARTED 0
+#define R_R_WAITING 108
+#define R_R_DOWNLOADING 109
+
+#define R_LM_PF_R_NO_FREE_NETWORK_CONNECTION_FOUND 110
+
+#define ALLOW_REQUESTS_TO_PROXY_FETCHERS 201
+#define DO_NOT_ALLOW_REQUESTS_TO_PROXY_FETCHERS 202
+#define ALLOW_REMOTE_NETWORKS 203
+#define DO_NOT_ALLOW_REMOTE_NETWORKS 204
+#define ALLOW_LOWER_PRIORITY_NETWORKS 205
+
+int decode_server_response(string server_response);
+
+#endif \ No newline at end of file
diff --git a/segget/segget.conf b/segget/segget.conf
index 26b39dd..21e28cd 100644
--- a/segget/segget.conf
+++ b/segget/segget.conf
@@ -76,7 +76,7 @@ max_tries=10
# Maximum value: 20
# Default:
# max_connections=10
-max_connections=20
+max_connections=3
# CURRENT_SPEED_TIME_INTERVAL_MSECS
# segget transfers may have bursty nature of their traffic. Therefore, while
@@ -172,7 +172,7 @@ provide_mirror_dir=./provide_mirror_dir
# patterns, no symlink will be provided to this distfile.
# Default:
# provide_mirror_files_restrict_list_on=0
-provide_mirror_files_restrict_list_on=1
+provide_mirror_files_restrict_list_on=0
[provide_proxy_fetcher_to_others]
# PROVIDE_PROXY_FETCHER_IP
@@ -216,7 +216,7 @@ provide_proxy_fetcher_port=3130
# network9_priority=0
network0_priority=10
network1_priority=9
-network2_priority=8
+network2_priority=0
network3_priority=0
network4_priority=0
network5_priority=0
diff --git a/segget/segget.cpp b/segget/segget.cpp
index 0754806..4cf5f9d 100644
--- a/segget/segget.cpp
+++ b/segget/segget.cpp
@@ -103,7 +103,8 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){
while(distfile_num<cur_pkg->distfile_count){
// if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){
if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){
- debug("Distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections");
+ debug("============================================= Distfile "
+ +cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections");
// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections");
// debug(" distfile_num:"+toString(distfile_num));
if (cur_pkg->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){
@@ -115,16 +116,21 @@ int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){
// debug(" segment_num:"+toString(segment_num));
// segments_in_progress[connection_num]=
// if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num);
- debug("Let's get segment status"+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status));
+ debug("Let's get segment status: "+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status));
if (cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){
- if ( ! cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){
+ if (0==cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){
return 0; // download started
};
+ // no success with this segment though it's waiting
+ // probably there will be a problem downloading next segment
+ // so let's go for the next distfile
+ // and return to this distfile later - next time pkg_choose_segment is called
+ break;
}else{
debug("status is not SWAITING - go for the next segment");
+ // haven't managed to provide this segment, go for the next one
+ segment_num++;
}
- // haven't managed to provide this segment, go for the next one
- segment_num++;
}
}else{
debug(" distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" has "
@@ -143,12 +149,12 @@ int choose_segment(uint connection_num){
try{
for (uint pkg_num=0; pkg_num<stats.pkg_count; pkg_num++){
// debug("pkg_num:"+toString(pkg_num));
- if (! pkg_choose_segment(Ppkg_array[pkg_num], connection_num)){
+ if (0==pkg_choose_segment(Ppkg_array[pkg_num], connection_num)){
return 0;
}
}
// download distfiles as a proxy-fetcher
- if (! pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){
+ if (0==pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){
return 0;
}
// for (uint array_item_num=0;array_item_num<pkg_count;array_item_num++){
@@ -193,19 +199,20 @@ int download_pkgs(){
U=1;
while (U) {
// Use free connections to download segments connections
+ debug("Entering connection activation cycle");
for (uint connection_num = 0; connection_num < settings.max_connections; ++connection_num) {
debug("connection_num:"+toString(connection_num));
if ( ! connection_array[connection_num].active){
-// if (
debug("connection is not active - choosing segment");
choose_segment(connection_num);
-// )
-// break;
+// // activate only one connection
+// break;
}
else{
debug("connection is active");
}
};
+ debug("Exit connection activation sycle");
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(cm, &U)){};
if (U) {
FD_ZERO(&R);
@@ -244,7 +251,7 @@ int download_pkgs(){
Tsegment *current_segment;
CURL *e = msg->easy_handle;
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &current_segment);
- uint connection_result=msg->data.result;
+ int connection_result=msg->data.result;
string result_msg_text="RESULT:"+toString(connection_result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment";
msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text);
curl_multi_remove_handle(cm, e);
diff --git a/segget/segment.cpp b/segget/segment.cpp
index 45116cf..cab6343 100644
--- a/segget/segment.cpp
+++ b/segget/segment.cpp
@@ -37,6 +37,25 @@ string statusToString(Tsegment_status the_status){
default :return "UNKNOWN STATUS";
}
}
+bool Tsegment::segment_verification_is_ok(){
+ try{
+ ulong downloaded_size;
+ ifstream file((settings.segments_dir+"/"+file_name).c_str(), ios::in|ios::binary);
+ ulong start = file.tellg();
+ file.seekg (0, ios::end);
+ ulong end = file.tellg();
+ downloaded_size = end - start;
+ debug("seg:"+toString(segment_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size));
+ file.close();
+ if (downloaded_size==segment_size){
+ return true;
+ }
+ }catch(...){
+ error_log("Error in segment.cpp: segment_verification_is_ok()");
+ }
+ return false;
+}
+
void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end){
try{
parent_distfile=prnt_distfile;
@@ -44,17 +63,9 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi
segment_size=range_end-seg_num*default_seg_size+1;
range=toString(seg_num*default_seg_size)+"-"+toString(range_end);
file_name="."+distfile_name+".seg"+toString(seg_num);
- ulong downloaded_size;
if (settings.get_resume()){
//check if downloaded
- ifstream file((settings.segments_dir+"/"+file_name).c_str(), ios::in|ios::binary);
- ulong start = file.tellg();
- file.seekg (0, ios::end);
- ulong end = file.tellg();
- downloaded_size = end - start;
- debug("seg:"+toString(seg_num)+" Dsize="+toString(downloaded_size)+" seg_size="+toString(segment_size));
- file.close();
- if (downloaded_size==segment_size){
+ if (segment_verification_is_ok()){
status=SDOWNLOADED;
debug("seg:"+toString(seg_num)+" Downloaded");
}
@@ -67,15 +78,11 @@ void Tsegment::set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfi
error_log("Error in segment.cpp: prepare_for_connection()");
}
}
-void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num){
+void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string url_address){
try{
// debug("NETWORK:"+toString(network_num)+(network_array[network_num].use_own_mirror_list_only_on ? " - LOCAL": " - REMOTE"));
- if (network_array[network_num].network_mode==MODE_LOCAL){
- url=network_array[network_num].benchmarked_mirror_list[mirror_num].url+parent_distfile->name;
- debug(" URL:"+url);
- }else{
- url=parent_distfile->url_list[mirror_num];
- }
+ // if MODE_LOCAL or MODE_PROXY_FETCHER
+ url=url_address;
msg_connecting(con_num,distfile_num, segment_num,"Downloading from "+url);
segments_in_progress[con_num]=this;
status=SDOWNLOADING;
diff --git a/segget/segment.h b/segget/segment.h
index af00f34..bdb6dcc 100644
--- a/segget/segment.h
+++ b/segget/segment.h
@@ -84,8 +84,9 @@ class Tsegment{
Tsegment(const Tsegment &L); // copy constructor
Tsegment & operator=(const Tsegment &L);
~Tsegment();
+ bool segment_verification_is_ok();
void set_segment(Tdistfile *prnt_distfile, uint seg_num, string distfile_name, ulong default_seg_size, ulong range_end);
- void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, uint mirror_num);
+ void prepare_for_connection(CURLM *cm, uint con_num, uint network_num, uint distfile_num, string url);
string get_file_name(){return file_name;};
int add_easy_handle_to_multi(CURLM *cm, uint network_num);
};