From: Andrew McNab Date: Mon, 3 Oct 2005 10:13:42 +0000 (+0000) Subject: gsexec fix X-Git-Tag: gridsite-core_R_1_1_12~46 X-Git-Url: http://scientific.zcu.cz/git/?a=commitdiff_plain;h=5eecab5861aa0efe932ab077c43b823422cc9bf9;p=jra1mw.git gsexec fix --- diff --git a/org.gridsite.core/CHANGES b/org.gridsite.core/CHANGES index ba03db6..02f6df6 100644 --- a/org.gridsite.core/CHANGES +++ b/org.gridsite.core/CHANGES @@ -1,3 +1,6 @@ +* Mon Oct 3 2005 Andrew McNab +- Fix to gsexec GRST_CRED_0/SSL_CLIENT_S_DN bug found + by Ian Stokes-Rees * Fri Sep 30 2005 Andrew McNab - Add SiteCast ping (NOP) support to htcp * Thu Sep 29 2005 Andrew McNab diff --git a/org.gridsite.core/doc/htcp.1 b/org.gridsite.core/doc/htcp.1 index f5ce2ac..a11eda8 100644 --- a/org.gridsite.core/doc/htcp.1 +++ b/org.gridsite.core/doc/htcp.1 @@ -1,12 +1,12 @@ .TH htcp 1 "September 2005" htcp "HTCP Manual" .SH NAME -.B htcp, htmv, htrm, htls, htll, htmkdir, htping +.B htcp, htmv, htrm, htls, htll, htmkdir, htfind, htping \- file transfers and queries via HTTP/HTTPS/SiteCast .SH SYNOPSIS .B htcp, htmv [\fIoptions\fR] \fISource-URL\fR[\fIs\fR] \fIDestination-URL\fR .br -.B htrm, htls, htll, htmkir +.B htrm, htls, htll, htmkir, htfind [\fIoptions\fR] \fITarget-URL\fR[\fIs\fR] .br .B htping @@ -17,9 +17,9 @@ is a client to fetch files or directory listings from remote servers using HTTP or HTTPS, or to put or delete files or directories onto remote servers using HTTPS. htcp is similar to scp(1), but uses HTTP/HTTPS rather than ssh as its transfer protocol. htcp can also use the HTCP protocol to query -GridSite via SiteCast. +HTTP(S) fileservers via SiteCast. -When talking to an HTTPS server, htcp can run "anonymously", with a +When talking to a fileserver with HTTPS, htcp can run "anonymously", with a standard X.509 user certificate and key, or with a GSI Proxy. This makes htcp very useful in Grid environments where many users have certificates and where jobs and users have access to GSI proxies. @@ -29,9 +29,9 @@ htcp supports the file:, http: and https: URL schemes as sources and destinations. If no scheme is given, the URL scheme is assumed to be file: and relative to the current directory if not an absolute path. -If multiple sources are given, they will be used in turn and the destination -must be a directory (directories are indicated by a trailing /) However, -source and destination cannot both refer to remote servers. +If multiple sources are given during a copy, they will be used in turn and +the destination must be a directory (directories are indicated by a trailing +/) However, source and destination cannot both refer to remote servers. .SH OPTIONS .IP "-v/--verbose" @@ -70,11 +70,20 @@ program as htmv has the same effect. .IP "--ping" .br -Query specified multicast groups with the HTCP NOP code. SiteCast enabled -servers will respond immediately with a NOP reply, and all of the responses -will be listed, with the round trip time in milliseconds. Any waiting times -specified in the --groups option will be ignored. Calling the program as -htping has the same effect. +Query specified multicast groups with the HTCP NOP ("No Operation") code. +SiteCast enabled servers will respond immediately with a NOP reply, and all +of the responses will be listed, with the round trip time in milliseconds. +Any waiting times specified in the --groups option will be ignored. Calling +the program as htping has the same effect. + +.IP "--find" +.br +Query specified multicast groups with the HTCP TST code. SiteCast enabled +servers will respond with TST replies if they have the files corresponding +to the given SiteCast target URL(s). All of the transfer URLs returned +will be listed. Waiting times specified in the --groups option will be used +to space out the multicast queries, but the program listens for responses +continuously. Calling the program as htfind has the same effect. .IP "--anon" .br @@ -117,14 +126,12 @@ transfer isn't possible, a normal HTTPS data transfer will be attempted. .IP "--groups " IP multicast groups to use for SiteCast queries. IP Groups is a comma -separated list of groups, in the format: -nnn.nnn.nnn.nnn[:port[:ttl[:seconds]]] The IP number must be -specified. If the UDP port isn't given, the default 777 will be used. -The IP time-to-live, ttl, controls how many networks the multicast packets may -pass through - the default, 1, limits packets to the local network. Multiple -groups may be specified, separated by commas. -If multiple groups are specified, then seconds is the time to wait before -making the next multicast - 1 second is the default. +separated list of groups, in the format: nnn.nnn.nnn.nnn:port[:ttl[:seconds]] +The IP number and port must be specified. The IP time-to-live, ttl, controls +how many networks the multicast packets may pass through - the default, 1, +limits packets to the local network. Multiple groups may be specified, +separated by commas. If multiple groups are specified, then seconds is the +time to wait before making the next multicast - 1 second is the default. .SH FILES .IP /tmp/x509up_uID diff --git a/org.gridsite.core/doc/htfind.1 b/org.gridsite.core/doc/htfind.1 new file mode 100644 index 0000000..11a60d1 --- /dev/null +++ b/org.gridsite.core/doc/htfind.1 @@ -0,0 +1 @@ +.so man1/htcp.1 diff --git a/org.gridsite.core/doc/htping.1 b/org.gridsite.core/doc/htping.1 new file mode 100644 index 0000000..11a60d1 --- /dev/null +++ b/org.gridsite.core/doc/htping.1 @@ -0,0 +1 @@ +.so man1/htcp.1 diff --git a/org.gridsite.core/src/Makefile b/org.gridsite.core/src/Makefile index 1ac72cf..935d6bb 100644 --- a/org.gridsite.core/src/Makefile +++ b/org.gridsite.core/src/Makefile @@ -309,6 +309,7 @@ install: apidoc ln -sf htcp $(prefix)/bin/htmkdir ln -sf htcp $(prefix)/bin/htmv ln -sf htcp $(prefix)/bin/htping + ln -sf htcp $(prefix)/bin/htfind cp -f gsexec $(prefix)/sbin cp -f mod_gridsite.so $(prefix)/lib/httpd/modules @@ -345,8 +346,8 @@ htcp-bin: htcp ../htcp-bin-$(PATCH_VERSION)/man/man1 cp -f ../doc/README.htcp-bin ../htcp-bin-$(PATCH_VERSION) cp -f htcp ../htcp-bin-$(PATCH_VERSION)/bin - cp -f ../doc/htcp.1 ../doc/htrm.1 ../doc/htls.1 ../doc/htll.1 \ - ../doc/htmkdir.1 ../doc/htmv.1 ../doc/htping.1 \ + cp -f ../doc/htcp.1 ../doc/htrm.1 ../doc/htls.1 ../doc/htmkdir.1 \ + ../doc/htll.1 ../doc/htmv.1 ../doc/htping.1 ../doc/htfind.1 \ ../htcp-bin-$(PATCH_VERSION)/man/man1 ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htls ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htll @@ -354,6 +355,7 @@ htcp-bin: htcp ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htmkdir ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htmv ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htping + ln -sf htcp ../htcp-bin-$(PATCH_VERSION)/bin/htfind cd ../htcp-bin-$(VERSION) ; tar zcvf ../htcp-$(VERSION).bin.tar.gz . rm -Rf ../htcp-bin-$(PATCH_VERSION) diff --git a/org.gridsite.core/src/gridsite.spec b/org.gridsite.core/src/gridsite.spec index 53d751f..613670e 100644 --- a/org.gridsite.core/src/gridsite.spec +++ b/org.gridsite.core/src/gridsite.spec @@ -91,12 +91,15 @@ rm -f %(echo ${MYPREFIX:-/usr})/share/doc/gridsite %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/bin/htmkdir %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/bin/htmv %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/bin/htping +%attr(-, root, root) %(echo ${MYPREFIX:-/usr})/bin/htfind %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htcp.1.gz %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htrm.1.gz %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htls.1.gz %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htll.1.gz %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htmkdir.1.gz %attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htmv.1.gz +%attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htping.1.gz +%attr(-, root, root) %(echo ${MYPREFIX:-/usr})/share/man/man1/htfind.1.gz %files gsexec %attr(4510, root, apache) %(echo ${MYPREFIX:-/usr})/sbin/gsexec diff --git a/org.gridsite.core/src/grst_htcp.c b/org.gridsite.core/src/grst_htcp.c index 55a32e6..ec9672a 100644 --- a/org.gridsite.core/src/grst_htcp.c +++ b/org.gridsite.core/src/grst_htcp.c @@ -240,6 +240,8 @@ int GRSThtcpMessageParse(GRSThtcpMessage *parsed, char *raw, int length) if ((parsed->opcode == GRSThtcpTSTop) && (parsed->rr == 0)) { + /* a TST request */ + /* point to start of data/auth in raw */ s = (GRSThtcpCountstr *) &(((GRSThtcpMessage *) raw)->method); @@ -276,6 +278,8 @@ int GRSThtcpMessageParse(GRSThtcpMessage *parsed, char *raw, int length) if ((parsed->opcode == GRSThtcpTSTop) && (parsed->rr == 1)) { + /* a TST response */ + /* point to start of data/auth in raw */ s = (GRSThtcpCountstr *) &(((GRSThtcpMessage *) raw)->method); diff --git a/org.gridsite.core/src/gsexec.c b/org.gridsite.core/src/gsexec.c index 292dcf3..c978fe4 100644 --- a/org.gridsite.core/src/gsexec.c +++ b/org.gridsite.core/src/gsexec.c @@ -530,6 +530,7 @@ int main(int argc, char *argv[]) uid_t httpd_uid; /* uid for AP_HTTPD_USER */ gid_t httpd_gid; /* uid for AP_HTTPD_GROUP */ char *mapping_type; /* suexec / X509DN / directory */ + char *grst_cred_0; /* GRST_CRED_0 */ char *map_x509dn; /* DN to use as pool acct. key */ char *map_directory; /* directory as pool acct. key */ @@ -640,8 +641,9 @@ int main(int argc, char *argv[]) else if (strcasecmp(mapping_type, "X509DN") == 0) { // log_err("X509DN mapping type\n"); - if ((map_x509dn = getenv("GRST_CRED_0")) == NULL) + if ((grst_cred_0 = getenv("GRST_CRED_0")) == NULL) map_x509dn = getenv("SSL_CLIENT_S_DN"); + else map_x509dn = index(grst_cred_0, '/'); if ((map_x509dn == NULL) || (map_x509dn[0] == '\0')) { @@ -650,8 +652,8 @@ int main(int argc, char *argv[]) exit(151); } - if (GRSTexecGetMapping(&target_uname, &target_gname, - GRST_EXECMAPDIR, map_x509dn) + if (GRSTexecGetMapping(&target_uname, &target_gname, + GRST_EXECMAPDIR, map_x509dn) != 0) { log_err("GRSTexecGetMapping() failed mapping \"%s\"\n", diff --git a/org.gridsite.core/src/htcp b/org.gridsite.core/src/htcp index 42bba47..8d187de 100644 Binary files a/org.gridsite.core/src/htcp and b/org.gridsite.core/src/htcp differ diff --git a/org.gridsite.core/src/htcp.c b/org.gridsite.core/src/htcp.c index 5b8e281..1219455 100644 --- a/org.gridsite.core/src/htcp.c +++ b/org.gridsite.core/src/htcp.c @@ -80,6 +80,7 @@ #define HTCP_MKDIR 6 #define HTCP_MOVE 7 #define HTCP_PING 8 +#define HTCP_FIND 9 #define HTCP_SITECAST_GROUPS 32 @@ -98,7 +99,9 @@ struct grst_stream_data { char *source; int gridhttp; int verbose; int timeout; - char *groups; } ; + char *groups; + int sitecast; + char *domain; } ; struct grst_index_blob { char *text; size_t used; @@ -611,12 +614,12 @@ int do_ping(struct grst_stream_data *common_data_ptr) &(sitecast_groups[igroup+1].quad3), &(sitecast_groups[igroup+1].quad4), &(sitecast_groups[igroup+1].port), - &(sitecast_groups[igroup+1].timewait), - &(sitecast_groups[igroup+1].ttl)); + &(sitecast_groups[igroup+1].ttl), + &(sitecast_groups[igroup+1].timewait)); if (ret == 0) break; /* end of list ? */ - if (ret < 4) + if (ret < 5) { fprintf(stderr, "Failed to parse multicast group " "parameter %s\n", p); @@ -683,7 +686,7 @@ int do_ping(struct grst_stream_data *common_data_ptr) if ((GRSThtcpMessageParse(&msg, response, response_length) == GRST_RET_OK) && - (msg.opcode == 0) && (msg.rr == 1) && + (msg.opcode == GRSThtcpNOPop) && (msg.rr == 1) && (msg.trans_id >= (int) start_timeval.tv_usec) && (msg.trans_id <= (int) (start_timeval.tv_usec + igroup))) { @@ -697,6 +700,144 @@ int do_ping(struct grst_stream_data *common_data_ptr) } } } + + return GRST_RET_OK; +} + +int do_finds(char *sources[], + struct grst_stream_data *common_data_ptr, int num) +{ + int isrc; + + int request_length, response_length, i, ret, s, igroup; + struct sockaddr_in srv, from; + socklen_t fromlen; +#define MAXBUF 8192 + char *request, response[MAXBUF], *p; + GRSThtcpMessage msg; + struct timeval start_timeval, wait_timeval; + struct grst_sitecast_group sitecast_groups[HTCP_SITECAST_GROUPS]; + fd_set readsckts; + + /* parse common_data_ptr->groups */ + + p = common_data_ptr->groups; + igroup = -1; + + for (igroup=-1; igroup+1 < HTCP_SITECAST_GROUPS; ++igroup) + { + sitecast_groups[igroup+1].port = GRST_HTCP_PORT; + sitecast_groups[igroup+1].timewait = 1; + sitecast_groups[igroup+1].ttl = 1; + + ret = sscanf(p, "%d.%d.%d.%d:%d:%d:%d", + &(sitecast_groups[igroup+1].quad1), + &(sitecast_groups[igroup+1].quad2), + &(sitecast_groups[igroup+1].quad3), + &(sitecast_groups[igroup+1].quad4), + &(sitecast_groups[igroup+1].port), + &(sitecast_groups[igroup+1].ttl), + &(sitecast_groups[igroup+1].timewait)); + + if (ret == 0) break; /* end of list ? */ + + if (ret < 5) + { + fprintf(stderr, "Failed to parse multicast group " + "parameter %s\n", p); + return CURLE_FAILED_INIT; + } + + ++igroup; + + if ((p = index(p, ',')) == NULL) break; + ++p; + } + + if (igroup == -1) + { + fprintf(stderr, "Failed to parse multicast group parameter %s\n", p); + return CURLE_FAILED_INIT; + } + + if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0) + { + fprintf(stderr, "Failed to open UDP socket\n"); + return CURLE_FAILED_INIT; + } + + /* loop through multicast groups since we need to take each + ones timewait into account */ + + gettimeofday(&start_timeval, NULL); + + for (i=0; i <= igroup; ++i) + { + if (common_data_ptr->verbose) + fprintf(stderr, "Querying multicast group %d.%d.%d.%d:%d:%d:%d\n", + sitecast_groups[i].quad1, sitecast_groups[i].quad2, + sitecast_groups[i].quad3, sitecast_groups[i].quad4, + sitecast_groups[i].port, sitecast_groups[i].ttl, + sitecast_groups[i].timewait); + + bzero(&srv, sizeof(srv)); + srv.sin_family = AF_INET; + srv.sin_port = htons(sitecast_groups[i].port); + srv.sin_addr.s_addr = htonl(sitecast_groups[i].quad1*0x1000000 + + sitecast_groups[i].quad2*0x10000 + + sitecast_groups[i].quad3*0x100 + + sitecast_groups[i].quad4); + + /* send off queries, one for each source file */ + + for (isrc=0; sources[isrc] != NULL; ++isrc) + { + GRSThtcpTSTrequestMake(&request, &request_length, + (int) (start_timeval.tv_usec + isrc), + "GET", sources[isrc], ""); + + sendto(s, request, request_length, 0, + (struct sockaddr *) &srv, sizeof(srv)); + + free(request); + } + + /* reusing wait_timeval is a Linux-specific feature of select() */ + wait_timeval.tv_usec = 0; + wait_timeval.tv_sec = sitecast_groups[i].timewait; + + while ((wait_timeval.tv_sec > 0) || (wait_timeval.tv_usec > 0)) + { + FD_ZERO(&readsckts); + FD_SET(s, &readsckts); + + ret = select(s + 1, &readsckts, NULL, NULL, &wait_timeval); + + if (ret > 0) + { + response_length = recvfrom(s, response, MAXBUF, + 0, &from, &fromlen); + + if ((GRSThtcpMessageParse(&msg, response, response_length) + == GRST_RET_OK) && + (msg.opcode == GRSThtcpTSTop) && (msg.rr == 1) && + (msg.trans_id >= (int) start_timeval.tv_usec) && + (msg.trans_id < (int) (start_timeval.tv_usec + num)) && + (msg.resp_hdrs != NULL) && + (GRSThtcpCountstrLen(msg.resp_hdrs) > 12)) + { + if (num > 1) printf("%s -> %.*s\n", + sources[msg.trans_id - (int) start_timeval.tv_usec], + GRSThtcpCountstrLen(msg.resp_hdrs) - 12, + &(msg.resp_hdrs->text[10])); + else printf("%.*s\n", + GRSThtcpCountstrLen(msg.resp_hdrs) - 12, + &(msg.resp_hdrs->text[10])); + } + } + } + + } return GRST_RET_OK; } @@ -1194,6 +1335,9 @@ int main(int argc, char *argv[]) {"ping", 0, 0, 0}, {"groups", 1, 0, 0}, {"timeout", 1, 0, 0}, + {"sitecast", 0, 0, 0}, + {"domain", 1, 0, 0}, + {"find", 0, 0, 0}, {0, 0, 0, 0} }; #if (LIBCURL_VERSION_NUM < 0x070908) @@ -1220,6 +1364,8 @@ int main(int argc, char *argv[]) common_data.groups = NULL; common_data.timeout = 0; + common_data.sitecast = 0; + common_data.domain = NULL; while (1) { @@ -1230,20 +1376,24 @@ int main(int argc, char *argv[]) if (c == -1) break; else if (c == 0) { - if (option_index == 1) common_data.cert = optarg; - else if (option_index == 2) common_data.key = optarg; - else if (option_index == 3) common_data.capath = optarg; - else if (option_index == 4) common_data.method = HTCP_DELETE; - else if (option_index == 5) common_data.method = HTCP_LIST; - else if (option_index == 6) common_data.method = HTCP_LONGLIST; - else if (option_index == 7) common_data.method = HTCP_MKDIR; - else if (option_index == 8) common_data.noverify = 1; - else if (option_index == 9) common_data.anonymous = 1; - else if (option_index ==10) common_data.gridhttp = 1; - else if (option_index ==11) common_data.method = HTCP_MOVE; - else if (option_index ==12) common_data.method = HTCP_PING; - else if (option_index ==13) common_data.groups = optarg; - else if (option_index ==14) common_data.timeout = atoi(optarg); + if (option_index == 1) common_data.cert = optarg; + else if (option_index == 2) common_data.key = optarg; + else if (option_index == 3) common_data.capath = optarg; + else if (option_index == 4) common_data.method = HTCP_DELETE; + else if (option_index == 5) common_data.method = HTCP_LIST; + else if (option_index == 6) common_data.method = HTCP_LONGLIST; + else if (option_index == 7) common_data.method = HTCP_MKDIR; + else if (option_index == 8) common_data.noverify = 1; + else if (option_index == 9) common_data.anonymous = 1; + else if (option_index ==10) common_data.gridhttp = 1; + else if (option_index ==11) common_data.method = HTCP_MOVE; + else if (option_index ==12) common_data.method = HTCP_PING; + else if (option_index ==13) common_data.groups = optarg; + else if (option_index ==14) common_data.timeout = atoi(optarg); + else if (option_index ==15) common_data.sitecast = 1; + else if (option_index ==16) { common_data.sitecast = 1; + common_data.domain = optarg; } + else if (option_index ==17) common_data.method = HTCP_FIND; } else if (c == 'v') ++(common_data.verbose); } @@ -1327,6 +1477,7 @@ int main(int argc, char *argv[]) else if (strcmp(executable,"htmkdir")==0) common_data.method=HTCP_MKDIR; else if (strcmp(executable,"htmv")==0) common_data.method=HTCP_MOVE; else if (strcmp(executable,"htping")==0) common_data.method=HTCP_PING; + else if (strcmp(executable,"htfind")==0) common_data.method=HTCP_FIND; } if (common_data.method == HTCP_PING) @@ -1340,6 +1491,7 @@ int main(int argc, char *argv[]) if ((common_data.method == HTCP_DELETE) || (common_data.method == HTCP_LIST) || + (common_data.method == HTCP_FIND) || (common_data.method == HTCP_MKDIR) || (common_data.method == HTCP_LONGLIST)) { @@ -1349,7 +1501,7 @@ int main(int argc, char *argv[]) printsyntax(argv[0]); return CURLE_URL_MALFORMAT; } - + sources = (char **) malloc(sizeof(char *) * (1 + argc - optind)); for (i=0; i < argc - optind; ++i) { @@ -1371,6 +1523,8 @@ int main(int argc, char *argv[]) anyerror = do_deletes(sources, &common_data); else if (common_data.method == HTCP_MKDIR) anyerror = do_mkdirs(sources, &common_data); + else if (common_data.method == HTCP_FIND) + anyerror = do_finds(sources, &common_data, argc - optind); else if (common_data.method == HTCP_LONGLIST) anyerror = do_listings(sources, &common_data, 1); else anyerror = do_listings(sources, &common_data, 0); @@ -1456,14 +1610,25 @@ int main(int argc, char *argv[]) return CURLE_URL_MALFORMAT; } - if ((common_data.method == HTCP_GET) && - ((strncmp(sources[i], "http://", 7) != 0) && - (strncmp(sources[i], "https://", 8) != 0))) + if (common_data.method == HTCP_GET) { - fputs("Cannot have both source and " + if ((strncmp(sources[i], "http://", 7) != 0) && + (strncmp(sources[i], "https://", 8) != 0)) + { + fputs("Cannot have both source and " "destination local (for now)\n\n",stderr); - printsyntax(argv[0]); - return CURLE_URL_MALFORMAT; + printsyntax(argv[0]); + return CURLE_URL_MALFORMAT; + } + +/* NEED TO CHECK common_data.domain MATCHES IF IT IS SET */ +/* + if (common_data.sitecast) + { + translate_sitecast_url(&sources[i], + sources[i], &common_data); + } +*/ } }