1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <stdio.h> 26 #include <stdlib.h> 27 #include <string.h> 28 #include <strings.h> 29 #include <sys/types.h> 30 #include <errno.h> 31 #include <syslog.h> 32 #include <unistd.h> 33 #include <sys/types.h> 34 #include <sys/socket.h> 35 #include <sys/time.h> 36 #include <netinet/in.h> 37 #include <arpa/inet.h> 38 #include <netdb.h> 39 #include <sys/stat.h> 40 #include <sys/sdt.h> 41 #include <signal.h> 42 #include <fcntl.h> 43 #include <libnvpair.h> 44 #include <libstmf.h> 45 #include <door.h> 46 #include <pthread.h> 47 #include <libscf.h> 48 #include <locale.h> 49 #include <sys/stmf_ioctl.h> 50 #include <sys/pppt_ioctl.h> 51 #include <libstmfproxy.h> 52 53 #define PPPT_NODE "/devices/pseudo/pppt@0:pppt" 54 #define USAGE "Usage: %s [-d][-f][-n nodeid] nodename\n" \ 55 "Note: nodename must be the same on both nodes\n" 56 57 58 /* 59 * static functions 60 */ 61 static void daemonInit(void); 62 static void killHandler(); 63 static int postMsg(uint_t nelem, uchar_t *aluaMsg); 64 65 66 /* 67 * globals 68 */ 69 void *t_handle; /* transport handle */ 70 char aluaNode[256]; /* one of the two alua peers */ 71 char myNode[256]; /* this hostname */ 72 int log_debug = 0; 73 int fore_ground = 0; 74 int proxy_hdl; 75 pt_ops_t *pt_ops; 76 pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER; 77 78 /* 79 * killHandler 80 * 81 * Terminates this process on SIGQUIT, SIGINT, SIGTERM 82 */ 83 /* ARGSUSED */ 84 static void 85 killHandler(int sig) 86 { 87 exit(0); 88 } 89 90 /* 91 * doorHandler 92 * 93 * Recieve data from the local proxy port provider and relay 94 * it to the peer node. 95 */ 96 /* ARGSUSED */ 97 void 98 doorHandler( 99 void *cookie, 100 char *args, 101 size_t alen, 102 door_desc_t *ddp, 103 uint_t ndid) 104 { 105 uint32_t result = 0; 106 107 if (ddp != NULL || ndid != 0) { 108 syslog(LOG_DAEMON|LOG_WARNING, 109 "descriptor passed to door %p %d", ddp, ndid); 110 result = EINVAL; 111 } 112 113 if (args == NULL || alen == 0) { 114 syslog(LOG_DAEMON|LOG_WARNING, 115 "empty message passed to door %p %d", args, alen); 116 result = EFAULT; 117 } 118 119 if (result == 0) 120 result = postMsg((uint_t)alen, (uchar_t *)args); 121 (void) door_return((char *)&result, sizeof (result), NULL, 0); 122 123 syslog(LOG_DAEMON|LOG_WARNING, "door_return FAILED %d", errno); 124 exit(errno); 125 } 126 127 static int 128 postMsg(uint_t nelem, uchar_t *aluaMsg) 129 { 130 uint32_t buflen; 131 uchar_t *buf; 132 int ret = 0; 133 int ns; 134 135 if (t_handle == NULL) { 136 syslog(LOG_DAEMON|LOG_WARNING, 137 "postMsg() no transport handle"); 138 exit(1); 139 } 140 141 buf = malloc(nelem + sizeof (buflen)); 142 143 buflen = htonl(nelem); /* length in network byte order */ 144 bcopy(&buflen, buf, sizeof (buflen)); 145 bcopy(aluaMsg, buf + sizeof (buflen), nelem); 146 147 (void) pthread_mutex_lock(&send_mutex); 148 ns = pt_ops->stmf_proxy_send(t_handle, buf, nelem + sizeof (buflen)); 149 (void) pthread_mutex_unlock(&send_mutex); 150 if (ns != nelem + sizeof (buflen)) { 151 ret = errno; 152 if (ret == 0) 153 ret = ENOTTY; /* something bogus */ 154 syslog(LOG_DAEMON|LOG_CRIT, "send() call failed: %d", ret); 155 } 156 free(buf); 157 return (ret); 158 } 159 160 /* 161 * Multi-thread the data path from the peer node to the local 162 * proxy port provider. During discover, there can be a large 163 * burst of messages from the peer node proportional to the number 164 * of LUs. Multiple threads allow these messages to be processed 165 * simultaneously. 166 */ 167 typedef struct pppt_drv_queue { 168 struct pppt_drv_queue *next; 169 uint32_t buflen; 170 uchar_t *buf; 171 } pppt_drv_queue_t; 172 173 pppt_drv_queue_t *pq_head = NULL; 174 pthread_mutex_t pq_mutex = PTHREAD_MUTEX_INITIALIZER; 175 pthread_cond_t pq_cond = PTHREAD_COND_INITIALIZER; 176 int pq_num_threads = 0; 177 int pq_avail_threads = 0; 178 179 /*ARGSUSED*/ 180 void * 181 push_to_drv(void *arg) 182 { 183 pppt_drv_queue_t *pq; 184 int rc; 185 186 (void) pthread_mutex_lock(&pq_mutex); 187 pq_num_threads++; 188 (void) pthread_mutex_unlock(&pq_mutex); 189 for (;;) { 190 (void) pthread_mutex_lock(&pq_mutex); 191 while (pq_head == NULL) { 192 pq_avail_threads++; 193 (void) pthread_cond_wait(&pq_cond, &pq_mutex); 194 pq_avail_threads--; 195 } 196 pq = pq_head; 197 pq_head = pq->next; 198 pq->next = NULL; 199 (void) pthread_mutex_unlock(&pq_mutex); 200 /* Relay the message to the local kernel */ 201 rc = stmfPostProxyMsg(proxy_hdl, (void *)pq->buf, pq->buflen); 202 if (rc != STMF_STATUS_SUCCESS) { 203 /* XXX die ? */ 204 syslog(LOG_DAEMON|LOG_CRIT, "ioctl failed - %d", errno); 205 } 206 free(pq->buf); 207 free(pq); 208 } 209 /*NOTREACHED*/ 210 return (NULL); 211 } 212 213 /* 214 * Receive data from peer and queue it up for the proxy driver. 215 */ 216 int message_count = 0; 217 static void 218 relay_peer_msg() 219 { 220 uint32_t buflen; 221 pppt_drv_queue_t *pq, *tmpq; 222 pthread_t tid; 223 int rc; 224 225 226 /* first receive the length of the message */ 227 if ((pt_ops->stmf_proxy_recv(t_handle, (uchar_t *)&buflen, 228 sizeof (buflen))) != sizeof (buflen)) { 229 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d", 230 errno); 231 exit(1); 232 } 233 234 pq = malloc(sizeof (*pq)); 235 pq->next = NULL; 236 pq->buflen = ntohl(buflen); 237 pq->buf = malloc(pq->buflen+4); 238 if (log_debug) { 239 syslog(LOG_DAEMON|LOG_DEBUG, 240 "recvMsg: size of buffer - %d", (int)pq->buflen); 241 } 242 243 if ((pt_ops->stmf_proxy_recv(t_handle, pq->buf, pq->buflen)) != 244 pq->buflen) { 245 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d", 246 errno); 247 exit(1); 248 } 249 250 /* Eat the first message from peer */ 251 if (message_count++ == 0) { 252 *(pq->buf+pq->buflen) = 0; 253 free(pq->buf); 254 free(pq); 255 return; 256 } 257 258 /* Queue the message to the driver */ 259 (void) pthread_mutex_lock(&pq_mutex); 260 if (pq_head == NULL) { 261 pq_head = pq; 262 } else { 263 /* add to the tail */ 264 tmpq = pq_head; 265 while (tmpq->next != NULL) 266 tmpq = tmpq->next; 267 tmpq->next = pq; 268 } 269 270 /* Make sure there is a thread to service this message */ 271 if (pq_avail_threads) { 272 /* wake an available thread */ 273 (void) pthread_cond_signal(&pq_cond); 274 (void) pthread_mutex_unlock(&pq_mutex); 275 } else { 276 /* no threads available, create a new thread */ 277 (void) pthread_mutex_unlock(&pq_mutex); 278 rc = pthread_create(&tid, NULL, push_to_drv, NULL); 279 if (rc != 0) { 280 syslog(LOG_DAEMON|LOG_WARNING, 281 "pthread_create() call failed: %d", rc); 282 if (pq_num_threads == 0) { 283 /* never created a thread */ 284 exit(rc); 285 } 286 } 287 } 288 } 289 290 /* 291 * Initialization for a daemon process 292 */ 293 static void 294 daemonInit(void) 295 { 296 pid_t pid; 297 int devnull; 298 299 if (fore_ground) 300 return; 301 302 if ((pid = fork()) < 0) { 303 syslog(LOG_DAEMON|LOG_CRIT, "Could not fork(). Exiting"); 304 exit(1); 305 } else if (pid != 0) { 306 /* 307 * XXX 308 * Simple approach for now - let the service go online. 309 * Later, set-up a pipe to the child and wait until the 310 * child indicates service is setup. 311 */ 312 exit(SMF_EXIT_OK); 313 } 314 315 (void) setsid(); 316 317 (void) chdir("/"); 318 319 (void) umask(0); 320 321 322 devnull = open("/dev/null", O_RDWR); 323 if (devnull < 0) { 324 syslog(LOG_DAEMON|LOG_CRIT, 325 "Failed to open /dev/null. Exiting"); 326 exit(1); 327 } 328 329 (void) dup2(devnull, STDIN_FILENO); 330 (void) dup2(devnull, STDOUT_FILENO); 331 (void) dup2(devnull, STDERR_FILENO); 332 (void) close(devnull); 333 } 334 335 void 336 daemon_fini(int rc) 337 { 338 /* 339 * XXX inform the parent about the service state 340 * For now, just exit on error. 341 */ 342 if (rc != 0) 343 exit(rc); 344 } 345 346 static int 347 open_proxy_driver() 348 { 349 int drv_door_fd; 350 int stmf_ret; 351 352 /* 353 * Create communication channel for the driver. 354 */ 355 if ((drv_door_fd = door_create(doorHandler, NULL, 0)) < 0) { 356 perror("door_create"); 357 syslog(LOG_DAEMON|LOG_DEBUG, 358 "could not create door: errno %d", errno); 359 return (SMF_EXIT_ERR_FATAL); 360 } 361 362 stmf_ret = stmfInitProxyDoor(&proxy_hdl, drv_door_fd); 363 if (stmf_ret != STMF_STATUS_SUCCESS) { 364 perror("pppt ioctl: door install"); 365 syslog(LOG_DAEMON|LOG_DEBUG, 366 "could not install door: errno %d", errno); 367 return (SMF_EXIT_ERR_FATAL); 368 } 369 370 return (SMF_EXIT_OK); 371 } 372 373 /* 374 * daemon entry 375 * 376 * parse arguments 377 * create resources to talk to child 378 * if !foreground 379 * daemonize, run as child 380 * open proxy driver 381 * install door in proxy driver 382 * create socket 383 * if server-side 384 * bind socket 385 * if !foreground 386 * inform parent things aok 387 * if parent 388 * exit(SMF_EXIT_OK) 389 * if server-side 390 * accept 391 * if client-side 392 * connect 393 * send hello 394 * recv hello 395 * loop on recieve 396 * XXX anyway to check in envp that we are started by SMF? 397 */ 398 int 399 main(int argc, char *argv[]) 400 { 401 struct sockaddr_in sin; 402 int rc; 403 struct sigaction act; 404 sigset_t sigmask; 405 int c; 406 int node = 0; 407 int node_override = 0; 408 int server_node = 0; 409 int server_match = 0; 410 extern char *optarg; 411 int stmf_ret; 412 413 (void) setlocale(LC_ALL, ""); 414 openlog("stmfproxy", LOG_PID, LOG_DAEMON); 415 (void) setlogmask(LOG_UPTO(LOG_INFO)); 416 417 while ((c = getopt(argc, argv, "dfn:")) != -1) { 418 switch (c) { 419 case 'd': 420 (void) setlogmask(LOG_UPTO(LOG_DEBUG)); 421 log_debug = 1; 422 break; 423 case 'f': 424 fore_ground = 1; 425 break; 426 case 'n': 427 node_override = 1; 428 node = atoi(optarg); 429 break; 430 default: 431 /* 432 * Should never happen from smf 433 */ 434 (void) fprintf(stderr, USAGE, argv[0]); 435 exit(SMF_EXIT_ERR_CONFIG); 436 break; 437 } 438 } 439 /* 440 * After the options, only the server argument should remain. 441 */ 442 if (optind != argc-1) { 443 (void) fprintf(stderr, USAGE, argv[0]); 444 exit(SMF_EXIT_ERR_CONFIG); 445 } 446 (void) strcpy(aluaNode, argv[optind]); 447 syslog(LOG_DAEMON|LOG_DEBUG, "aluaNode %s", aluaNode); 448 if (gethostname(myNode, 255)) { 449 perror("gethostname"); 450 exit(1); 451 } 452 if ((inet_aton(aluaNode, &sin.sin_addr)) == 0) { 453 /* 454 * Not ipaddr, try hostname match. 455 */ 456 server_match = (strcmp(aluaNode, myNode)) ? 0 : 1; 457 } else { 458 /* 459 * see if this is our ip address 460 */ 461 (void) fprintf(stderr, "Sorry, cannot use ip adress format\n"); 462 } 463 if (server_match) { 464 server_node = 1; 465 if (!node_override) 466 node = 1; 467 } 468 469 470 /* 471 * Allow SIGQUIT, SIGINT and SIGTERM signals to terminate us 472 */ 473 act.sa_handler = killHandler; 474 (void) sigemptyset(&act.sa_mask); 475 act.sa_flags = 0; 476 477 /* Install the signal handler */ 478 (void) sigaction(SIGQUIT, &act, NULL); 479 (void) sigaction(SIGINT, &act, NULL); 480 (void) sigaction(SIGTERM, &act, NULL); 481 (void) sigaction(SIGHUP, &act, NULL); 482 483 /* block all signals */ 484 (void) sigfillset(&sigmask); 485 486 /* unblock SIGQUIT, SIGINT, SIGTERM */ 487 (void) sigdelset(&sigmask, SIGQUIT); 488 (void) sigdelset(&sigmask, SIGINT); 489 (void) sigdelset(&sigmask, SIGTERM); 490 (void) sigdelset(&sigmask, SIGHUP); 491 492 (void) sigprocmask(SIG_SETMASK, &sigmask, NULL); 493 494 /* time to go backstage */ 495 daemonInit(); 496 497 if ((rc = open_proxy_driver()) != 0) 498 daemon_fini(rc); 499 500 if ((rc = stmf_proxy_transport_init("sockets", &pt_ops)) != 0) 501 daemon_fini(rc); 502 503 /* 504 * Establish connection 505 * 506 * At this point, the parent has exited and the service 507 * is online. But there are no real proxy services until 508 * this connect call succeeds. That could take a long time if 509 * the peer node is down. 510 */ 511 t_handle = pt_ops->stmf_proxy_connect(server_node, aluaNode); 512 if (t_handle == NULL) { 513 syslog(LOG_DAEMON|LOG_WARNING, 514 "socket() call failed: %d", errno); 515 exit(1); 516 } 517 518 /* The first message is a greeting */ 519 (void) postMsg((uint_t)strlen(myNode)+1, (uchar_t *)myNode); 520 /* Read the greeting from peer node */ 521 relay_peer_msg(); 522 /* 523 * Set the alua state in stmf. No need to keep 524 * the device open since the proxy driver has a reference. 525 */ 526 stmf_ret = stmfSetAluaState(B_TRUE, node); 527 if (stmf_ret != STMF_STATUS_SUCCESS) { 528 syslog(LOG_DAEMON|LOG_CRIT, "stmf ioctl failed - %x", stmf_ret); 529 exit(1); 530 } 531 532 /* service is online */ 533 daemon_fini(0); 534 535 /* 536 * Loop relaying data from the peer daemon to the local kernel. 537 * Data coming from the local kernel is handled asynchronously 538 * by the door server. 539 */ 540 for (;;) { /* loop forever */ 541 relay_peer_msg(); 542 } 543 } 544