/*===================================================================== Copyright (C) 2011-2012 Alain BENEDETTI aka Zakhar @ ubuntu.fr This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published byr the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see =====================================================================*/ /*====================================================================================\ | Function: | |========== | | This program mounts a Freebox Revolution NAS (provided by French ISP: Free) | | on a local directory using fuse. | | | | The mount is read-only | | We can mount a local Freebox (via http://mafreebox.freebox.fr) or a distant one | | provided this has been allowed in the administrative console. For distant access | | you must use the public IP and the port that has been assigned in the console. | | | | For local mount, this is of poor interest as you can already achieve the same | | mount with cifs (slow) or curlftpfs (faster), which do also provide writing. | | | | For distant mount, this is a UNIQUE solution, as there are no file system (that I | | know of) capable of mounting a "Freebox" (or similar devices exposing only an | | HTML interface). | | | |-------------------------------------------------------------------------------------| | Usage: | |======= | | See description in the usage() function. | | Basically there are a few parameters than the usual for a mount: | | - what to mount (in this case a URI to our Freebox) | | - where to mount (a local directory) | | - Options for verbosity, help, version, password, config. file,... | | - Remaining options are handled by fuse (like -d, -o blahblah,...) | | | |-------------------------------------------------------------------------------------| | Tested: Ubuntu Lucid (10.04.3 x64) | |======== | | | |-------------------------------------------------------------------------------------| | Version: 0.6.2 | |========= | | Note that using the -V/--version option would get you this version plus the fuse | | version and it's libraries, as the -V option is passed along to fuse. | | | |-------------------------------------------------------------------------------------| | Date: 2012-01-08 | |====== | | | |-------------------------------------------------------------------------------------| | Author: Alain BENEDETTI aka Zakhar | |======== | | | |-------------------------------------------------------------------------------------| | History: | |========= | | fork of version 0.5.06 + multi-thread (essentially!) | | - 0.6.2: removed chain list of response semaphores to allieavate synchronization. | |2012-2-11 Now replaced by a __thread variable. Does also simplify code! | | - 0.6.1: debug messages are now out of 'normal' compiled version (-O3), to get debug| | 2012 messages, we must add -DDEBUG to the compile line (and usually also -g to | | 01-08 get the symbols). With this modification, the normal 'fast' version has a | | smaller footprint, and is a touch faster. | | | |-------------------------------------------------------------------------------------| | Compiling instructions: | |======================== | | At the moment, you don't even need a make file. A one-liner instruction works: | | | | $ `curl-config --cc --cflags --libs` `pkg-config fuse --cflags --libs`\ | | -o fbxrofs fbxrofs.c | | | | add -g if you want debug symbols, and -Wall to check warnings change/add code. | | | | Dependencies: libcurl and fuse, so you must add to you development environment the | | 2 corresponding packages (for example on Lucid they are called: | | libcurl4-gnutls-dev and libfuse-dev, it will install libraries, | | headers and developers documentation) | | | | The counterpart of this easy compiling (no make!) is that all the code resides in | | this single file, no .h, no breakdown in logical functions. As long as is remains | | manageable, I'll try to keep it that way: plain and simple 1 file thing! | | To help manage the code, you got a summary, and code is divided into "pages" | | which numbers are in the summary. | | | |-------------------------------------------------------------------------------------| | General Notes: | |=============== | | As it is tightly related to a device provided by a French ISP, and if someone | | (beside me!) uses this software directly, (s)he most certainly is French or | | understands French, I could have commented in French! | | Nevertheless, as it could be used as template for similar functions, I decided to | | stick to English comments. | | | | As a matter of fact, you could achieve the same functionnality with almost any | | http server provided you have: | | - a way to connect (password & so on) | | - a way to list directories (with characteristics such as size of files, dates...) | | - a way to download a file inside the mounted tree... | | ... including ranges, because we need it for efficient reads! | | | | Freebox V6 (Revolution), the Free (French ISP) device offers all these functions. | | It features an administrative Website that allow you, through a browser, to see | | all the directories tree, the files, and download them. Most of this "website" is | | using JQuery/JSON. The informations about directories, files, etc is thus available| | in JSON format. The download part also features ranges, although it is not imple- | | mented in the HTML site, it works. | | So, like curlftpfs (helped me for fuse as fuse is badly documented!) we can use | | curl to acces the Freebox and provide the result to the fuse callbacks. | | | | The program uses now multi-thread. Although we could suppose single-thread would | | not impair reactivity a lot it did! Doing a simple ls while copying a file sort of | | 'stuck' the ls. VLC was stuttering on streamed music, because VLC intermingles | | 'getattrs' with 'reads'. Now with multi-thread, all is much more smoother. | | Do note that the logging process uses it's own lock and is used in ALL the callbacks| | so if you wish better performance, don't log!.. Without log, the only locking | | function is 'read' because it needs to do a curl to get data from the Freebox. | | So with no log, 'readdir' and 'getattrs' can run in parallel we no limit. | | | | For the sake of simplicity we use one "shortcut": the directory tree is scanned | | once when the filesystem is mounted. That is because the Freebox is not usually | | an active NAS where lot of write/new files happens. And as we are read-only, we | | don't add/change files ourself. If you wish to refresh the directory tree, you | | need to un-mount (fusermount -u) and mount again. | | This limitation might be lifted in subsequent versions (if needed!). | | | |-------------------------------------------------------------------------------------| | Disclaimer: | |============ | | I'm not affiliated with Free, Iliad or any of it's subsidiaries. I'm merely a | | customer (and most of my family too because it's the best offer so far in France). | | | | I didn't consult with Free representative prior to this develoment. But as I'm only | | using what is publicly available (the administrative web interface) I can't see any | | logical reason why this should be forbidden. In fact, this program might generate | | less traffic than the regular Web interface, because it does not download the HTML | | CSS, JS and images (they are useless for our purpose) but only the JSON. | | Although compared to a simple download, the traffic is slightly higher because we | | we use partial downloads (Ranges) which generates a few percents of overhead. | | On a 50MB download, I measured less than 4% penalty: 8:19 instead of 8:01 for | | a regular download. Could probably cancel most of this penalty with the read-ahead | | technique. | | | | Please, do also note that the code of the Web interface has not been made public. | | Free had to make public their modifications on GPL stuff they use (kernel, network | | stacks, etc...) after a long trial, but the Web interface being their own creation | | plus probably some LGPL/BSD/MIT -like JQuery they indeed use- that do no obligation | | to have to publish any of you added code. | | So all the communication functions here use "what we can see". It means that: | | - there might be special conditions that trigger behaviour I never observed. | | - Free can change without prior notice the way the administrative Web works. | | All that could affect the program correct behaviour. | | Nevertheless, this Read-Only file system uses only 3 features from the Freebox: | | Login, Read a directory, download a file. So even if it changes, modifications in | | the code should not be too difficult... unless some of the features are removed! | | | |-------------------------------------------------------------------------------------| | Security: | |========== | | About security, Free did the worst work ever! The admin password goes on the wire | | unencrypted, and the same for all transfers. I won't comment more about that you'll | | find plenty places where it is detailed. | | So, the best you can do to avoid major risk, is use the admin web (or this program) | | "inside" the Free network. Of course, if you use it locally with ethernet cable to | | your freebox there is little risk! If you want to use it to access a "remote" | | Freebox, your best shot is to use a PC that is also connected through Free | | (or Alice). | | There is nothing this program, or any other program can do to improve that. | | Due to this huge security breach, you should avoid using this program in the | | autostart (such as rc.local, gnome startup, etc...) unless it is on a desktop you | | never move, ethernet-connected to the Free network as stated above. | | For the password handling, see the different ways to give the password in the | | comments page 9. The more "secure" (and less friendly) is the keyboard input. | | The less secure (and most user-friendly) is to set it in the default config file. | |-------------------------------------------------------------------------------------| | Architecture: | |============== | | -First design: a rather simple first design was to get chunks of data when fuse ask | | for them. Fuse, when using 'read' on a file, tells you: give me N bytes of file | | 'path' at the offset X. And you do exactly that, sending a 'Range' matching the | | offset and length on that file. | | => Benefits: it is straightforward and simple, no programming complexity. | | => Drawback: it has a noticeable overhead, especially on sequential reads (copy | | rsync, streaming -although in this case it is not an issue as you must stream | | at a rate that is lower to your bandwith). The overhead is due to 2 factors | | 1) there are headers in and out for each chunk, which take up a fraction of | | the bandwidth. On big files, the kernel reads 128K blocks. The bandwith | | that we must save is the upload from the Freebox. This scheme takes up | | around 300 bytes for each block of 128K in reply. So around 0.25% | | 2) but more importantly, there is a latency doing so, as we 'pay' for a round | | trip to the server for each block we read. | | Global performance impact on a 50Mb file (ADSL upload at 1Mbps) is around 4%. | | Of course it depends on your 'ping'. In my use case, I have a 40ms ping to the | | server, which means that I'm losing 40ms every request. A request on big copies| | being 128K, which is exactly 1sec at 1Mbps, the calcultation fits: we lose | | 40ms each 1s (time needed for a 128K block), which is exactly a 4% loss. | | => Drawback: reads are 'serialised'. They are anyway serialised at some point | | because, for example, you probably have only 1 network link to the server, so | | bytes will flow 'serialised'!.. But when both doing 'big' copies and stream | | music, you end up having 128K read serialised. Assuming you still have enough | | bandwith to stream without delay, the serialisation migth create delays and | | the user could experience bad music/video playback, because (in my use case) | | as blocks take 1s to read, you might end up waiting for data blocks to play. | | Of course a simple workaround is to instruct your player to buffer enough data | | (which you should do anyway as we are network-streaming)... but anyway, a | | 'parallel' reading scheme would be much safer. | | | | -Better design: of course, there is no 100% better scheme in 100% situations! But | | yet, using a consumer/worker pattern, with a careful algorithm to when we should | | use 'Ranges' and when we shouldn't, gives better results in many situations. | | In this design, when fuse fires a read callback (the consumer is our callback), | | the callback makes a request with that read block to a reader. The reader is | | selected 'per file' at the first read request following the 'open', and then does | | not change till the file is closed (release for fuse). The chosen reader is the | | 'best' possible: a free (or uninitialised) reader when available, if none we try | | to find one not doing sequential reads, then we select the one with less files. | | => Benefits: in many situation we can do 'sequential' reads, thus avoiding the | | overheads explained above. That is because many actions triggers such type | | of reads: cp, rsync, copy with a graphical tool, any many other more use case| | => Drawbacks: the program is noticeably more complex, as it involves asynchronous | | features in order to be able to continue reading when we have answered to | | the fuse request. | | => Drawbacks: there are situations where we read data and then discard them. | | that is because we 'guessed' the read would be sequential but it turned to | | be something 'random' or an 'out of order' sequence (although we manage to | | fix some of those), or because the user decided to stop the reading program. | | The issue here is that we consumed some bandwith uselessly. It can be of no | | consequence if the bandwith is not the bottleneck but could slow things down | | if we need 100% bandwith and too much 'wrong guess' occur. | | => Drawbacks: of course... buffers take memory! We allocate 128K (the kernel max | | read size) for each reader. | | Options like -r 1 (one reader only) or --no-cache take care of those drawbacks | | and still do some 'optimisations' compared to the simpler design. | |-------------------------------------------------------------------------------------| | TODO: | | - (low) It is not illegal for ext3/4 to have filenames with 0x1 to 1F chars although| | it is difficult to produce such filenames :). Try it and see if something breaks! | | - (low) 'bash escaping' of strings while reading the config file as someone might | | need both ' and " inside a password string (which is impossible without escaping) | \====================================================================================*/ /*====================================================================================\ | Page: 1 | | |==========/ | | SUMMARY: PAGE | | -------- ___________________________ ---- | | _____________________________________________/ SECTION: 'utils & params' \________ | | Prologue (defines, types, constants, etc) ................................. 2 | | lprintf(), l_printf() ..................................................... 3 | | malloc_or_die(), realloc_or_die(), alloc_cat() ............................ 4 | | usage() ................................................................... 5 | | add_fuse_arg(), scan_arguments() .......................................... 6 | | JSONescape(), URIescape() ................................................. 7 | | check_URI(), check_mountpoint() ........................................... 8 | | read_password(), get_password() ........................................... 9 | | | | _____________________________________________ | | ___________________________/ SECTION: 'curl utils & tree initialisation' \________ | | curl_easy_setopt_or_die(), curl_easy_perform_or_die() ..................... 10 | | write_trash(), write_alloc(), parse_status_code(), parse_status_range() ... 11 | | is_login_ok(), server_login(), server_reconnect() ......................... 12 | | get_JSON_num(), copy_JSON_str() ........................................... 13 | | len_JSON_str(), get_JSON_type(), compare_nodes() .......................... 14 | | store_server_tree() ....................................................... 15 | | free_server_tree() ........................................................ 16 | | | | _______________________ | | _________________________________________________/ SECTION: 'fuse utils' \________ | | compare_dir(), compare_file(), search_dir() ............................... 20 | | fill_attr() ............................................................... 21 | | split_path() .............................................................. 22 | | _________________________ | | _______________________________________________/ SECTION: 'thread utils' \________ | | pthread_mutex_lock_or_die(), pthread_mutex_unlock_or_die() | | sem_init_or_die(), sem_wait_or_die(), sem_post_or_die() ................... 25 | | AllocElement(), FreeElement(), DestroyElements() .......................... 26 | | AllocFS() ................................................................. 27 | | Respond(), store_data(), fill_request() ................................... 28 | | | | ___________________________ | | _____________________________________________/ SECTION: 'fuse callbacks' \________ | | fbx_getattr_wrapper(), fbx_getattr() ...................................... 30 | | fbx_readdir_wrapper(), fbx_readdir() ...................................... 31 | | fbx_open_wrapper(), fbx_open() ............................................ 32 | | fbx_release_wrapper(), fbx_release() ...................................... 33 | | //Prototypes of functions// ............................................... 34 | | fbx_destroy_wrapper() ..................................................... 35 | | fbx_init() ................................................................ 36 | | | | ___________________________________________ | | _____________________________/ SECTION: 'The Asynchronous Reading stuff' \________ | | server_logout(), log_stats(), end_reader() ............................... 40 | | fbx_read_wrapper() ....................................................... 41 | | write_move() ............................................................. 42 | | compute_path() ........................................................... 43 | | fbx_read() ............................................................... 44 | | init_reader() ............................................................ 45 | | start_reader() ........................................................... 46 | | async_reader() ........................................................... 47 | | | | __________________ | | ______________________________________________________/ SECTION: 'main' \________ | | fuse_operations ........................................................... 50 | | main() .................................................................... 51 | | | \====================================================================================*/ /*====================================================================================\ | Page: 2 | | |==========/ | | Prologue: | | You'll find here | | - defines | | - includes | | - "constant strings": error messages, search patterns, default values, etc... | | - structures and type definitions and declaration | | - 'semi-constants' (set once during initiasation then immutables). | | - and our global variable declaration used to keep state of our filesystem | | | \====================================================================================*/ // The FUSE API has been changed a number of times. So, our // code needs to define the version of the API that we assume. // As of this writing, the most current API version is 26 #define FUSE_USE_VERSION 26 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*---------------------- Constants & strings constants ------------------------------*/ #define DEFAULT_READERS_NUMBER 4 /* Default number of Reader Threads */ #define MAX_READERS_NUMBER 32 /* Unreasonable having more readers!*/ #define RA_BUFFER_SIZE 128 * 1024 /* Size of the Read-Ahead buffer */ #define PROG_NAME "fbxrofs" #define VERSION_STRING "Version 0.6.2" #define PASSWORD_PROMPT "Mot de passe : " #define HTTPE_OK 200 /* This is HTTP response StatusCodes*/ #define HTTPE_FOUND 302 /* See RFC 2616 for more information*/ #define HTTPE_PARTIAL_CONTENT 206 #define HTTPE_FORBIDDEN 403 #define HTTPE_NOT_FOUND 404 #define COMMAND_STOP (1 << 0) /* 0x01 */ #define COMMAND_FILL_BUFFER (1 << 1) /* 0x02 */ /* Critical or Error level messages. ____________________________ */ #define ALLOC_ERR_MSG "Erreur d'allocation mémoire\n" #define MUTEX_LOCK_FAILED_ERR_MSG "Erreur de verrouillage de mutex\n" #define MUTEX_UNLOCK_FAILED_ERR_MSG "Erreur de déverrouillage de mutex\n" #define SEM_INIT_FAILED_ERR_MSG "Initialisation de sémaphore échouée.\n" #define SEM_WAIT_FAILED_ERR_MSG "Erreur sémaphore: sem_wait.\n" #define SEM_POST_FAILED_ERR_MSG "Erreur sémaphore: sem_post.\n" #define SEM_DESTROY_ERR_MSG "Destruction du sémaphore échouée\n" #define THREAD_CREATION_FAILED_ERR_MSG "Création de thread échouée.\n" #define READ_THREAD_STOP_FAILED_ERR_MSG "Erreur à l'arrêt du fil de lecture [%u].\n" #define CURL_ERROR_ERR_MSG "Erreur curl %d\nMessage : %s\n" #define CURL_SET_OPT_ERR_MSG "Erreur %d sur curl_easy_setopt(,%s,)\nMessage : %s\n" #define INVALID_URI_SCHEME_ERR_MSG\ "L'URI %s n'est pas valide, elle doit commencer par http://\n" #define ILL_FORMED_URI_ERR_MSG "L'URI %s est mal formée.\n" #define STAT_FAILED_OR_DIR_DOES_NOT_EXIST_ERR_MSG \ "Erreur stat() ou répertoire %s inexistant.\n" #define NOT_A_DIRECTORY_ERR_MSG "%s n'est pas un répertoire.\n" #define INCORRECT_PASSWORD_ERR_MSG "Mot de passe incorrect.\n" #define ILL_FORMED_JSON_RESPONSE_ERR_MSG "Réponse JSON mal formée : %s\n" #define UNKNOWN_CURL_OPT_MSG "Option inconnue" #define HTTP_JSON_ERROR_MSG "Erreur HTTP: %d\nJSON: %s\n" #define CURL_NO_DATA_RECEIVED_MSG "Erreur : aucune de donnée reçue !\n" #define CURL_INIT_FAILED_MSG "Erreur d'initialisation de curl" /* Warnig level messages. _______________________________________ */ /* Note, 1st to 3rd messages are fatal. They are classified as */ /* Warnings so that the logger does not exit and we can print the */ /* short usage message to recommend using help option. */ #define INCORRECT_ARG_COUNT_WARN_MSG "Nombre d'arguments incorrect\n" #define BAD_LOG_LEVEL_SPECIFICATION_WARN_MSG \ "Mauvaise spécification du niveau de journalisation : `%s'\n" #define BAD_READERS_NUMBER_WARN_MSG \ "Mauvaise spécification du nombre de lecteurs : `%s'\n" #define CONFIG_FILE_NOT_FOUND_WILL_TRY_WITH_DEFAULT_WARN_MSG \ "Le fichier de configuration: `%s' n'existe pas.\n\ Tentative avec le fichier de configuration par défaut.\n" #define NO_READ_POSSIBLE_WARN_MSG\ "ATTENTION: aucune lecture ne sera possible !\n\ La racine sur la Freebox: `%s' contient un '?' (bug connu)\n" #define NOTHING_ACCESSIBLE_WARN_MSG\ "Aucun fichier ni répertoire accessible : montage annulé.\n" #define JSON_ERR_ON_DIR_WARN_MSG "Erreur JSON: %s" #define JSON_ERR_ON_DIR_CONSEQUENCE_WARN_MSG \ "Les fichiers du répertoire : `%s' seront inaccessibles\n\n" #define BUG_FBX_WARN_MSG "ATTENTION : %s contient un '?'.\n\ Compte tenu d'un bug de la Freebox, vous ne pourrez pas le télécharger, ainsi que \ tout son contenu s'il s'agit d'un répertoire.\n" #define DECONNEXION_FAILED_WARN_MSG "Déconnexion du lecteur [%u] échouée http=%d\n" /* Notice level messages. _______________________________________ */ #define FBX_READ_TREE_MSG "Lecture de l'arborescence de la Freebox...\n" #define TREE_SUMMARY_COUNT_MSG "%d fichiers, %d répertoires, %d liens\r" #define FBX_READ_TREE_SUCCESS_MSG "Lecture Freebox OK\n\n" #define TREE_SUMMARY_TOTAL_MSG "Total: %d fichiers, %d répertoires, %d liens\n" #define STARTING_FUSE_MSG "Démarrage de Fuse\n\n" #define DAEMON_BACKGROUNG_INIT_MSG "Initialisation en arrière-plan du démon.\n" #define DISCONNECT_READER_SUCCESS_MSG "Déconnexion du lecteur [%u] réussie\n" #ifdef __i386__ #define STAT_BYTES_READ_MSG "Octets lus : %lld\n" #define STAT_BYTES_CACHED_MSG "Octets cachés : %lld\n" #define STAT_BYTES_LOST_MSG "Octets perdus : %lld\n" #else /* !__i386__ */ #define STAT_BYTES_READ_MSG "Octets lus : %ld\n" #define STAT_BYTES_CACHED_MSG "Octets cachés : %ld\n" #define STAT_BYTES_LOST_MSG "Octets perdus : %ld\n" #endif /* __i386__ */ #define STAT_FILES_READ_MSG "Fichiers lus : %ld\n" #define STAT_READS_COUNT_MSG "Nb. lectures : %ld\n" #define STAT_ABORTED_READS_MSG "Lec. arrêtées : %ld\n" #define STAT_TOTAL_MSG "Statistiques totales :\n" #define INVALID_COOKIE_MSG "Cookie échu ou supprimé |%u|: reconnexion.\n" #define RECONNEXION_SUCCESS_MSG "Reconnexion automatique réussie.\n" /* Info level messages. _________________________________________ */ #define NO_BUFFER_OPTION_MSG \ "Option no-buffer : buffers et lecture séquentielle désactivés.\n" #define LIST_DIR_MSG "Listage répertoire=%s\n" #define NEW_FILE_ALLOCATED_MSG "Nouvelle allocation : fichier {%u}.\n" #define FUSE_OPEN_MSG "{%u} Open (%s,) ==> (%d)\n" #define FUSE_RELEASE_MSG "{%u} Release ==> (%d)\n" #define READER_INITIALIZED_MSG "Initialisation du lecteur (%u) OK\n" #define READ_THREAD_STARTED_MSG "Fil de lecture asynchrone [%u] démarré.\n" #define READ_THREAD_STOPPED_MSG "Fil de lecture asynchrone [%u] arrêté.\n" #define PARAM_READ_OK_MSG "Lecture des paramètres OK :\n" #define FREEBOX_URI_MSG "fbxURI=%s\n" #define FREEBOX_ROOT_MSG "fbxRoot=%s\n" #define LOCAL_MOUNTPOINT_MSG "mountpoint=%s\n\n" #define FILE_ASSOCIATED_IDLE_READER_MSG "{%u} <=associé=> [%d] (libre)\n" #define FILE_ASSOCIATED_BUSY_READER_MSG "{%u} <=associé=> [%d] (occupé)\n" #ifdef __i386__ #define TREE_DETAIL_MSG "%d:(%s/%lld) %s\n" #else /* !__i386__ */ #define TREE_DETAIL_MSG "%d:(%s/%ld) %s\n" #endif /* __i386__ */ #define READER_STATS_DETAIL_MSG "Statistiques du lecteur [%u] :\n" /* Debug level messages. ________________________________________ */ #ifdef DEBUG #define JSON_DUMP_MSG "%d:%s\n" #define NODE_MEMORY_NEEDED_MSG "On a %d noeuds, et on a besoin de %ld mémoire\n" #define FUSE_GETATTR_MSG "Getattr(%s,) ==> (%d)\n" #ifdef __i386__ #define RESPOND_READ_MSG ">>> [%u]{%u} ==> (%d) %d\n" #define FUSE_READDIR_MSG "Readdir(%s,,,%lld,) ==> (%d)\n" #define BUFFER_CLEANUP_AT_RELEASE_MSG \ "Buffer nettoyé (%d octets à %lld) à la fermeture de `%s'\n" #define CACHE_HIT_DETAIL_MSG \ "<<< ***Cache hit*** [%d]{%u}: %d octets sur %d à %lld\n" #define REVERSED_SEQUENCE_MSG "[%d]{%u}: Séquence inversée %lld/%lld\n" #define SEQ_READ_CONTINUATION_MSG "<<< [%u]{%u} (...suite...) , %d octets à %lld\n" #define BUFFER_BYTES_LOST_MSG "...[%u]{%u}: %d octets perdus, `%s' à %lld\n" #define CONTINUATION_AFTER_BUFFERING_MSG \ "<<< [%u]{%u} (...suite buffer...) , %d bytes at %lld\n" #define STOP_BUFFERING_ON_NON_SEQ_READ_MSG \ "[%u]{%u} Abort bufferisation : requête non séquentielle ! (%lld/%lld){%u}\n" #define READ_AHEAD_MSG \ "<<<(--) [%u]{%u} (...lecture en avance...) , %d octets à %lld\n" #define GLOBING_2_REQUESTS_MSG \ "[%u]{%u} 2 requêtes englobées %d + %d octets à %lld\n" #define INCOMING_REQUESTS_MSG "<<< [%u]{%u}, %d octets à %lld\n" #else /* !__i386__ */ #define RESPOND_READ_MSG ">>> [%u]{%u} ==> (%d) %ld\n" #define FUSE_READDIR_MSG "Readdir(%s,,,%ld,) ==> (%d)\n" #define BUFFER_CLEANUP_AT_RELEASE_MSG \ "Buffer nettoyé (%ld octets à %ld) à la fermeture de `%s'\n" #define CACHE_HIT_DETAIL_MSG \ "<<< ***Cache hit*** [%d]{%u}: %ld octets sur %ld à %ld\n" #define REVERSED_SEQUENCE_MSG "[%d]{%u}: Séquence inversée %ld/%ld\n" #define SEQ_READ_CONTINUATION_MSG "<<< [%u]{%u} (...suite...) , %ld octets à %ld\n" #define BUFFER_BYTES_LOST_MSG "...[%u]{%u}: %ld octets perdus, `%s' à %ld\n" #define CONTINUATION_AFTER_BUFFERING_MSG \ "<<< [%u]{%u} (...suite buffer...) , %ld bytes at %ld\n" #define STOP_BUFFERING_ON_NON_SEQ_READ_MSG \ "[%u]{%u} Abort bufferisation : requête non séquentielle ! (%ld/%ld){%u}\n" #define READ_AHEAD_MSG \ "<<<(--) [%u]{%u} (...lecture en avance...) , %ld octets à %ld\n" #define GLOBING_2_REQUESTS_MSG \ "[%u]{%u} 2 requêtes englobées %ld + %ld octets à %ld\n" #define INCOMING_REQUESTS_MSG "<<< [%u]{%u}, %ld octets à %ld\n" #endif /* __i386__ */ #define READ_AHEAD_REQUEST_MSG \ "[%d]{%u}: Demande de remplissage du buffer par avance\n" #define SEQ_READ_STOPPED_MSG \ "[%u]{--} Arrêt lecture : requête non séquentielle ou fichier est fermé.\n" #define START_BUFFERING_MSG "[%u]{%u}: Bufferisation...\n" #define STOP_BUFFERING_ON_FILE_CLOSED_MSG \ "[%u]{--}: Arrêt bufferisation car le fichier est fermé.\n" #define STOP_BUFFERING_ON_BUFFER_FULL_MSG "[%u]{%u}: Fin bufferisation.\n" #define BUFFER_FULL_MSG "[%u]{%u}: Buffer rempli.\n" #define READ_AHEAD_IGNORED_ON_OTHER_REQUESTS_MSG \ "[%u] Commande 'fill buffer' ignorée car on a d'autres requêtes.\n" #define IMMEDIATE_SEQ_READ_MSG "... [%u]{%u} === lecture séquentielle immédiate.\n" #define SEQUENTIAL_READ_MSG "... [%u]{%u} === lecture séquentielle.\n" #define SEQUENTIAL_READ_FROM_SLOW_MSG "... [%u]{%u} === séq. depuis 'slow'.\n" #define RANDOM_READ_MSG "... [%u]{%u} === lecture aléatoire.\n" #endif /* __DEBUG__ */ static char *pJSONHeader="Content-Type: application/json; charset=utf-8"; static char *pDefConfig="/.config/freebox.conf"; /* Default config file name */ static char aHTTPScheme[]="http://"; /* URI MUST start with that */ static char aIdentity[]="http://freebox:"; /* Search string for identity in URI*/ static char aPassVar[]="fbxPassword="; /* Search str password in conf files*/ static char aHTTPCode[]="HTTP/1.1 "; /* to extract HTTP response code */ /* Search patterns to scan the JSON string we get from a read-dir */ static char aJSONOK[]="\"jsonrpc\":\"2.0\",\"result\":"; /* => test if all is OK */ static char aMime[]="\"mimetype\":\""; static char aName[]="\"name\":\""; static char aMod[] ="\"modification\":"; static char aType[]="\"type\":\""; static char aSize[]="\"size\":"; /* Values for the "type" field of the JSON string */ static char aDirTypeValueJSON[] ="dir"; static char aFileTypeValueJSON[]="file"; static char aLinkTypeValueJSON[]="link"; /* Human readable name for the enum nodeType */ static char *aTypes[4]={"Répertoire","Fichier","Lien","Autre"}; /*---------------------- ??printf strings that differ 32b/64b -----------------------*/ #ifdef __i386__ #define FORMAT_RANGE_HEADER "Range: bytes=%lld-%lld" #else /* !__i386__ */ #define FORMAT_RANGE_HEADER "Range: bytes=%ld-%ld" #endif /* __i386__ */ /*---------------------- Stucts, enum, typdefs --------------------------------------*/ typedef enum /* Type of elements the Freebox */ { /* handles. tOTHER is in case we */ tDIR, /* have a type I didn't see during */ tFILE, /* my tests */ tLINK, tOTHER } nodeType; typedef enum /* Type of read detected on files */ { /* to optimize network traffic. */ rSEQ, /* SEQuential read (best scenario!) */ rSLOW, /* SLOW sequential (like streaming) */ rRAND, /* RANdom read (bad scenario!) */ rINIT, /* Value after initialisation */ rCALC /* and goes to CALC on 2nd hit. */ } readType; struct FSNode /* This is the basic structure of */ { /* each element in our FileSystem */ struct FSNode *parent; /* Link to parent dir, self for root*/ char *pName; /* ... names are self-explanatory */ time_t modification; nodeType type; off_t size; union { struct { struct FSNode *pFirstNode; /* First node of Dir */ int nNodes; /* number of nodes in that Dir */ int nDirs; /* number of dirs in that Dir */ } child; /* For Dir: struct to chain on child*/ char *pMimetype; /* For Regular files: mimetype */ /* For Soft links: at the moment we can't do anything, because the Freebox interface gives us no way to have more information that "it is a link" */ } detail; }; typedef struct FSNode FSNode; typedef struct /* Generic pointer/lenght structure */ { /* used with strings when we can't */ void * pb; /* or do not wish to use the O */ size_t cb; /* terminating mark. Sometimes we */ size_t max; /* need a max size of the buffer. */ } pbcb; typedef struct _fs { struct _fs *pNextFile; FSNode *pNode; int iReader; int iFS; } fs_t; typedef struct _rq { sem_t *pSemResp; /* Semaphore to put the response */ fs_t *pFS; /* Links this read to an opened handle */ pbcb pbcb; off_t offset; char *pPath; size_t sizeRead; /* Response to the current request being served : */ int error; /* -Size we read, and error (0 means OK) */ struct _rq *pNextReq; } request_t; typedef struct { off_t bytesRead; off_t bytesCached; off_t bytesLost; long nFS; long nReads; long nAborts; } stat_t; typedef struct /* Structure for the readers. */ { CURL *curl; /* Pointer to the curl object used for all curl operations */ CURLcode res; /* curl return code for our macro (during fuse operations) */ char curlErrBuf[CURL_ERROR_SIZE]; /* And curl text-error buffer */ int statusCode;/* HTTP statuscode (RFC 2616) of all HTTP request made */ sem_t *pSemPutReq;/* Semaphore to put request */ request_t *pReq; /* Request... when not buffering! */ fs_t *pLastFS; /* For detection of sequential reading, we need the last */ off_t lastOffset;/* fs we read on, and the last offset for SEQ read detect */ off_t endRead; /* End of the read req. (used in fill_buff) */ pthread_mutex_t lThis;/* Lock for this reader's private data. (the fields below) */ readType readType; /* Type of read used */ int command; /* Used to send special commands to the reader */ pbcb pbcb; /* Read-Ahead pbcb (buffer+lenght+max-lenght) */ fs_t *pFS; /* FS of the read-ahead buffer */ off_t offset; /* offset of the read-ahead buffer. */ int nFiles; request_t *pFirstReq; /* Queue of pending requests for that Reader. */ stat_t stats; /* Statistic counters. */ } reader_t; /*---------------------- Semi-constants ---------------------------------------------*/ /* The variable below are set during the initialisation process and used "read-only" */ /* in the rest of the process */ char *pFbxRoot= "/"; char *pPostReadRoot= "filename="; int sPostReadRoot; int logLevel=LOG_NOTICE; uid_t uid; gid_t gid; int argcFuse; char **argvFuse; bool fForeground= false; bool fInitDone= false; bool fDaemon= false; bool fNoCache= false; int maxReaders= DEFAULT_READERS_NUMBER; /* These are variables prepared for CURL: URIs and posts */ char *pFbxLoginURI; char *pFbxLogoutURI; char *pFbxReadDirURI; char *pFbxDownloadURI; char *pFbxPasswordPost=NULL; /*---------------------- Our Global state variable ----------------------------------*/ /* Here we store the GLOBAL (stateful) data that will be useful during the fuse */ /* operations in the callbacks. */ /* As we write here (for example the curl object is written by curl during network */ /* calls) it is not thread-safe. So, as we are multi-thread we have to "mutex" the */ /* places where these variables are used, or use several of them, which will */ /* translate to having several connection to the Freebox. The limitation for remote */ /* access being the ADSL upload speed, at the moment, this is not much better than */ /* "mutex", and probably not worth the complexity. */ /* In order to know when to "mutex" all the "unsafe" variables are enclosed in the */ /* struct below. Note that at the moment FSRoot is a semi-constant and could be */ /* placed outside the struct, we kept it here in case we add a "refresh directories" */ /* later, which will make it truely "not-constant". */ /* Lock for multithreading critical parts */ static pthread_mutex_t lAlloc= PTHREAD_MUTEX_INITIALIZER; static fs_t *pFirstFS; /* Structures for opened files. */ static int nFS; /* Number of 'FS' allocated */ static FSNode FSRoot; /* Root pointer of the Freebox filesystem tree */ static reader_t *pReaders; static pthread_t *pThreadRead; /*====================================================================================\ | Page: 3 | | |==========/ | | Function: lprintf(), l_printf() | |========== | | | | Description: | |============= | | Utility function that logs our messages and exit on severe errors. | | | | The log function is now more 'intelligent'! | | | | As long as we are not yet in fuse, we print at the console. This is denote by the | | flag fInitDone that is initially false, and put to 'true' just before calling | | fuse_main. | | We do also detect if fuse was daemonized or run foreground (options -f or -d or | | -o debug). If so, we continue displaying our messages to console. | | When message are displayed to console, all that is Notice or less important goes | | to stdout, and all that is Warning or more important goes to stderr. | | When the init is done, and we are daemonised, the messages go to the daemon syslog | | | | The exit occurs | | - for all messages Error or more severe during the init. | | - for all messages Critical or more severe after the init. | | It is so because we don't want to start running if there is an error the user can | | easyly correct (wrong parameter for example). But once we run, we try to | | recover as much as we can from errors, and exit only we there is nothing we can | | do to recover. | | | | There is also a main 'loglevel'. If the user specifies a 'loglevel' (eitherwise it | | default to Notice), all message of the severity of more are output. So the user | | can select the verbosity from debug (extremely verbose) to critical (almost no | | message). Levels are those of syslog (see the man). We do not have messages | | at Alert and Emerg level, as we run at userspace, we should be able to disrupt | | the kernel itself with such high importance messages. Thus if you chose these | | levels, you won't get any message (could be a desidered feature!). | | NOTE: be aware that at Debug level, it is VERY verbose. If this level is selected | | you should run with the option -f to have messages on the console (or redirected | | to a file) as fuse with stay foregroud. If you don't there are chances you will | | flood you daemon syslog with thousands of messages!.. | | | | The macro is used as a speed shortcut. When the condition is expressed as 'true' | | (most of the time!) the 3 conditions on the test fall down to one, the other 2 | | are resolved at compile time. Then if the level is less severe to what the user | | wants, the variables for the function won't be pushed uselessly down the stack. | | | | The 'Check' macro can be used so that gcc checks the parameters against printf, | | this can avoid nasty GP Faults as it won't detect with our function. | | | \====================================================================================*/ bool l__printf( int level, char *pFormat, ... ) { va_list ap; va_start( ap, pFormat ); if (level <= logLevel) { if (fDaemon) vsyslog( level, pFormat, ap); else vfprintf( (( level <= LOG_WARNING ) ? stderr : stdout ), pFormat, ap ); } if ( ( fInitDone && level <= LOG_CRIT) || ( !fInitDone && level <= LOG_ERR ) ) exit(EXIT_FAILURE); return true; } #define l_printf(level, cond, pFormat, ...) \ ( \ ( ((level <= LOG_ERR) || (level <= logLevel)) && (cond) ) \ ? \ l__printf( level, pFormat, ##__VA_ARGS__) \ : \ false \ ) #ifdef CHECK_LPRINTF #define lprintf(level, cond, pFormat, ...) printf(pFormat, ##__VA_ARGS__) #else /* !__CHECK_LPRINTF__ */ #define lprintf l_printf #endif /* __CHECK_LPRINTF__ */ #ifdef DEBUG #define debug( cond, pFormat, ...) \ ( \ ( (LOG_DEBUG <= logLevel) && (cond) ) \ ? \ l__printf( LOG_DEBUG, pFormat, ##__VA_ARGS__) \ : \ false \ ) #else /* !__DEBUG__ */ #ifdef CHECK_LPRINTF #define debug( cond, pFormat, ...) printf(pFormat, ##__VA_ARGS__) #else /* !__CHECK_LPRINTF__ */ #define debug( cond, pFormat, ...) true #endif /* __CHECK_LPRINTF__ */ #endif /* __DEBUG__ */ /*====================================================================================\ | Page: 4 | | |==========/ | | Functions: malloc_or_die, realloc_or_die, alloc_cat | |=========== | | | | Description: | |============= | | These are utility functions for easy memory allocation. | | Basically, what the first 2 do, is alloc/realloc memory, and exit with a failure | | message if it does not succeed. This allow us, on the rest of the code, to call | | these allocating functions without having to test the returned buffer everywhere. | | | | The third function is a sort of replacement for the + operator on strings. | | The function concatenates the given strings in a newly allocated buffer which | | pointer is returned. It takes a variable number of pointers to valid null termi- | | nated strings. The last pointer must be NULL, indicating that there are no more | | strings to concatenate. From the caller point of view this function does never | | fail (no need to test returned buffer for NULL), because if it failed in the | | memory allocation (only part it could fail) the program exits as we use | | internally our 'malloc_or_die' | | | \====================================================================================*/ void * malloc_or_die( size ) size_t size; { void *p; p=malloc(size); lprintf( LOG_CRIT, p==NULL, ALLOC_ERR_MSG ); return p; } void * realloc_or_die( p, size ) void *p; size_t size; { p=realloc( p, size ); lprintf( LOG_CRIT, p==NULL, ALLOC_ERR_MSG ); return p; } /*-----------------------------------------------------------------------------------*/ /* The concatenation occurs in 2 phases to avoid realloc which could be time consuming as the same strings could be copied over and over on reallocated buffers. */ void * alloc_cat( char *pStr, ... ) { char *p, *r; size_t len=0; va_list ap; /* Phase 1 : we compute the total length of all strings */ va_start( ap, pStr ); while( ( p= va_arg( ap, char * ) ) != NULL) { len+=strlen( p ); } va_end( ap ); /* Phase 2 : we allocate and move the strings to the buffer */ r= malloc_or_die( strlen( pStr ) + len + 1 ); strcpy( r, pStr ); va_start(ap, pStr); while( ( p= va_arg( ap, char * ) ) != NULL) { strcat( r, p ); } va_end(ap); return r; } /*====================================================================================\ | Page: 5 | | |==========/ | | Function: usage() | |========== | | | | Description: | |============= | | Classicly this function displays messages about the usage of this program. | | When it is misused (status not being EXIT_SUCCESS) a short message is displayed | | on stderr advising to invoke the program with -h to get help. | | When help is required (-h/--help) the entire help message is displayed on the | | standard output. Note that in this case, the option been passed also to fuse | | you do also get the fuse general help message. | | Whatever the status passed, the program will exit after the messages are displayed | | (after fuse messages display when invoked with status=EXIT_SUCCESS) | | | \====================================================================================*/ void usage( status ) int status; { lprintf( LOG_ERR, status != EXIT_SUCCESS, "Saisissez « %s --help » pour plus d'informations.\n", PROG_NAME ); printf ("Usage: fbx [OPTION] Freebox Où\n\n"), fputs ("\ Monte la freebox sur l'emplacement spécifié\n\ \n\ Options:\n\ -c, --config=CONF_FICH Fichier configuration (def.: /.config/freebox.conf)\n\ -p, --password=PASSWRD Mot de passe de connexion à la Freebox\n\ -l, --log=LOG_NIVEAU Niveau de journalisation 0 à 7 (déf.: 5, cf syslog)\n\ -n, --no-buffer Aucune bufferisation ni lecture séquentielle\n\ -r, --readers=NOMBRE Nombre de lecteurs 1 à 32 (déf.: 4)\n\ -h, --help Affiche cette aide\n\ -V, --version Affiche la version de ce programme\n\ \n\ Freebox:\n\ À distance : http://[freebox:password@]ip[:port]\n\ Localement : http://[freebox:password@]mafreebox.freebox.fr\n\ \n\ Où: répertoire où monter la Freebox\n\ \n\ Mot de passe : il peut être spécifié de 4 façons.\n\ Par ordre de priorité décroissante on considère:\n\ - 1) option `-p'\n\ - 2) passé dans l'URI\n\ - 3) dans le fichier de configuration (spécifié ou par défaut)\n\ - 4) demandé à l'utilisateur\n\ \n\ Fichier de configuration\n\ Doit contenir une ligne de la forme :\n\ fbxpassword=\"Mot_De_Passe\"\n\ Si la commande est précédée d'un #, elle est considérée comme un commentaire\n\ et n'est pas prise en compte.\n\ \n", stdout); } /*====================================================================================\ | Page: 6 | | |==========/ | | Functions: add_fuse_arg(), scan_arguments() | |=========== | | | | Description: | |============= | | This couple of functions is used to: scan the arguments passed to the program | | (2nd function) and prepare the remaining arguments for fuse (1st function) | | | | scan_arguments handles its own arguments and gives back an argc/argv structure for | | unhandled arguments that are intended to Fuse (via calls to arg_fuse_arg) | | - Specific arguments of our program: -p, -c, -l | | - Common arguments : -h/--help, -V/--version | | - Specific Fuse arguments : -d -f -s -o arg (-o has a mandatory argument) | | The specific to our program and common arguments are "consumed" by the function to | | set the relevant flag/parameter. | | The common and specific to fuse arguments, are prepared for fuse. | | The fonction returns 0 (false) when a common argument is used, because as it is | | only --help or --version, we do not want the real process to take place but just | | messages from our program and from fuse. Otherwise the function returns 1 (true) | | meaning we want the real stuff !.. On error, there is not return but an exit with | | the relevant message. | | | | add_fuse_arg puts the option/optarg couple (although optarg can be NULL) in the | | argc/argv structure (pointers) we use to prepare fuse arguments. | | We take care of cases when we have only an option, only an optarg, both or none! | | NOTE: we cannot easily guess how much arguments are going to be needed for fuse | | not even use the argc of our program as a maximum. That is because it is | | legal to "pack" short options like -dso arg, and as we use the getopt_long | | facility, this results is several loops. So the arguments pushed to fuse | | can be more than argc. Thus add_fuse_arg is bound to use realloc method. | | | \====================================================================================*/ void add_fuse_arg( option, optarg ) char option; char *optarg; { char *p; int count=0; if ( option != '\0' ) count++; if ( optarg != NULL ) count++; if ( count != 0 ) { argvFuse= realloc_or_die( argvFuse, (argcFuse + count ) * sizeof(char **) ); if ( option != '\0' ) { p= argvFuse[argcFuse++]= malloc_or_die( 3 ); *(p++)= '-'; *(p++)= option; * p = '\0'; } if ( optarg != NULL ) { p= argvFuse[argcFuse++]= malloc_or_die( strlen(optarg) + 1 ); strcpy( p, optarg ); } } } /*-----------------------------------------------------------------------------------*/ int scan_arguments( argc, argv, ppFbxURI, ppConfigFile, ppMountpoint) int argc; char **argv; char **ppFbxURI; char **ppConfigFile; char **ppMountpoint; { int c; int option_index = -1; /* getopt_long stores the option index here. */ struct option long_options[]={ {"config" ,required_argument, NULL, 'c'}, {"password" ,required_argument, NULL, 'p'}, {"log" ,required_argument, NULL, 'l'}, {"readers" ,required_argument, NULL, 'r'}, {"no-buffer", no_argument, NULL, 'n'}, {"help" , no_argument, NULL, 'h'}, {"version" , no_argument, NULL, 'V'}, {NULL , 0, NULL, 0 } }; argvFuse= malloc_or_die( sizeof(char **) ); argvFuse[0]= argv[0]; argcFuse= 1; while (1) { c = getopt_long (argc, argv, "c:p:l:r:nhVdfso:", long_options, &option_index); if (c == -1) break; switch (c) { case 0: /* If this option set a flag we abort... ... because we have no such options. */ abort (); break; case 'd': case 'f': fForeground= true; case 's': add_fuse_arg( c, NULL); break; case 'o': if ( strcmp(optarg, "debug") == 0 ) fForeground= true; add_fuse_arg( c, optarg); break; case 'c': *ppConfigFile=optarg; break; case 'p': pFbxPasswordPost=malloc_or_die( strlen(optarg) + 1 ); strcpy(pFbxPasswordPost, optarg); break; case 'h': add_fuse_arg( c, NULL); usage(EXIT_SUCCESS); return 0; case 'n': fNoCache= true; lprintf( LOG_INFO, true, NO_BUFFER_OPTION_MSG ); break; case 'l': { unsigned int level; if ( sscanf( optarg, "%u", &level ) == 0 || level > LOG_DEBUG ) { lprintf( LOG_WARNING, true, BAD_LOG_LEVEL_SPECIFICATION_WARN_MSG, optarg ); usage(EXIT_FAILURE); } else logLevel= level; } break; case 'r': { unsigned int nReaders; if ( sscanf( optarg, "%u", &nReaders ) == 0 || nReaders == 0 || nReaders > MAX_READERS_NUMBER ) { lprintf( LOG_WARNING, true, BAD_READERS_NUMBER_WARN_MSG, optarg ); usage(EXIT_FAILURE); } else maxReaders= nReaders; } break; case 'V': add_fuse_arg( c, NULL); printf( "%s: %s\n", PROG_NAME, VERSION_STRING ); return 0; case '?': /* getopt_long already printed an error message. */ usage(EXIT_FAILURE); default: abort (); } } /* We must have exactly 2 arguments left : 1- The Freebox address (IP address [:port] or myfreebox.freebox.fr for local) 2- The mount point path */ if (optind != argc - 2) { lprintf( LOG_WARNING, true, INCORRECT_ARG_COUNT_WARN_MSG ); usage(EXIT_FAILURE); } *ppFbxURI = argv[optind ]; *ppMountpoint= argv[optind+1]; add_fuse_arg( 'o', "ro"); /* Force read-only filesystem */ add_fuse_arg( '\0', *ppMountpoint); /* Push the Mountpoint to fuse */ return 1; /* This means we are in a "normal" situation, all OK, proceed normally */ } /*====================================================================================\ | Page: 7 | | |==========/ | | Function: JSONescape(), URIescape() | |========== | | | | Description: | |============= | | Utility functions for escaping strings. | | | | JSONescape escapes 2 character: " and \. Each one is escaped with a leading \. | | If there is something to escape, memory is allocated to hold the new escaped | | string. Otherwise, we return the same pointer we got as parameter. | | Thus, to free memory, the caller can test if both pointer are different. | | | | URIescape escapes 3 character: %, & and +. Each one is escaped with a % and the 2 | | hexa digit code of the character. Also, '?' returns NULL because currently there is| | a bug on the Freebox and it is impossible to download file having '?' in their name| | Another difference with the other function, URIescape prefixes the escaped string | | with the given prefix. Thus allocation always happens. When there is nothing to | | escape, this is (more or less) equivalent to the utility alloc_cat, escaping added | | when necessary. | | | \====================================================================================*/ char * JSONescape( pPath ) char *pPath; { int l= 0; int n= 0; char *p, *q; for( p=pPath; *p; p++ ) { if ( *p== '"' || *p== '\\' ) n++; l++; } if (n) { p= q= malloc_or_die( l + n + 1 ); for( ; *pPath; pPath++ ) { if ( *pPath== '"' || *pPath== '\\' ) *(q++)='\\'; *(q++)= *pPath; } *q='\0'; return p; } else return pPath; } /*-----------------------------------------------------------------------------------*/ char * URIescape( pPath, pPrefix, sPrefix ) char *pPath; char *pPrefix; int sPrefix; { int l=0; int n=0; char *p, *q; for( p= pPath; *p; p++) { if ( *p=='?' ) return NULL; /* Freebox bugs on namefiles containing a ? */ if ( *p=='%' || *p=='&' || *p=='+' ) n++; l++ ; } p= malloc_or_die( sPrefix + l + 2 * n + 1 ); memcpy( p, pPrefix, sPrefix ); q= p + sPrefix; if (n) { for( ; *pPath; pPath++) { if ( *pPath=='%' || *pPath=='&' || *pPath=='+' ) { sprintf(q,"%%%.2X", *pPath); q+=3; } else *(q++)=*pPath; } *q='\0'; } else strcpy( q, pPath ); return p; } /*====================================================================================\ | Page: 8 | | |==========/ | | Functions: check_URI(), check_mountpoint() | |=========== | | | | Description: | |============= | | This couple of functions checks the 2 main parameters: the URI of our Freebox to | | mount, and the local directory where we want to mount it. | | | | check_URI | | - checks that the URI starts with http:// | | - if there is an "indentity", in the form freebox:password@, extracts the password | | (unless password is already specified with -p option which has higher priority), | | then strip the identity string from the URI. | | - then if a path was specified (eg http://10.20.30.40:44444/Disk/MyPath) save it | | and extract it so that we have 2 variables: root of the server, and our mount | | starting point. | | | | check_mountpoint | | - checks that the path exists. | | - checks that the path is a directory. | | | \====================================================================================*/ void check_URI( pFbxURI ) char *pFbxURI; { char *p; /* The URI provided MUST start with the scheme, in this case http:// */ lprintf( LOG_ERR, strncmp( pFbxURI, aHTTPScheme, sizeof(aHTTPScheme)-1 ) != 0, INVALID_URI_SCHEME_ERR_MSG, pFbxURI ); /* Identity handling: save password (unless -p) and strip identity */ if ( strncmp( pFbxURI, aIdentity, sizeof(aIdentity)-1 ) == 0 ) { char *pEndPassword; int passwordLen; pEndPassword= strchr( pFbxURI, '@' ); lprintf( LOG_ERR, pEndPassword == NULL || strlen(pEndPassword + 1) == 0, ILL_FORMED_URI_ERR_MSG, pFbxURI ); /* Here we found a correct aIdentity, we extract password (if not already set by -p option) and remove the indentity (freebox:password@) from URI */ passwordLen= pEndPassword - ( pFbxURI + sizeof(aIdentity)-1 ); if ( pFbxPasswordPost == NULL ) { pFbxPasswordPost= malloc_or_die( (passwordLen + 1) ); memcpy( pFbxPasswordPost, (pFbxURI + sizeof(aIdentity)-1), passwordLen ); *( pFbxPasswordPost + passwordLen ) = '\0'; } memmove( pFbxURI + sizeof(aHTTPScheme)-1, /* Here we use memmove because */ pEndPassword + 1, /* dest & source overlap and in */ strlen( pEndPassword + 1 ) + 1 /* this case memcpy is unsafe */ ); } /* Now we compute the root of our mount on the Freebox. This is the path after the IP (or symbol) address. If no path, we want / of the Freebox */ if ( ( p= strchr( pFbxURI + sizeof(aHTTPScheme), '/' )) != NULL ) { int l; l= strlen( p ); if ( l > 1 ) /* Only way we have l=1 is when only '/' */ { /* So here we have more than '/' (root) */ if ( *( p + l - 1 ) == '/' ) /* If trailing '/' we remove it */ *( p + l - 1 )= '\0'; pFbxRoot= alloc_cat( p, NULL ); *p= '\0'; } } if ( strcmp( pFbxRoot, "/" ) != 0 ) { pPostReadRoot= URIescape( pFbxRoot, pPostReadRoot, strlen( pPostReadRoot ) ); lprintf( LOG_WARNING, pPostReadRoot== NULL, NO_READ_POSSIBLE_WARN_MSG, pFbxRoot); } if ( pPostReadRoot != NULL ) sPostReadRoot= strlen( pPostReadRoot ); } /*-----------------------------------------------------------------------------------*/ void check_mountpoint( pMountpoint ) char *pMountpoint; { /* We check that pMountpoint is a directory */ struct stat sbuf; lprintf( LOG_ERR, stat(pMountpoint, &sbuf) != EXIT_SUCCESS, STAT_FAILED_OR_DIR_DOES_NOT_EXIST_ERR_MSG, pMountpoint ); lprintf( LOG_ERR, ! S_ISDIR(sbuf.st_mode), NOT_A_DIRECTORY_ERR_MSG, pMountpoint ); } /*====================================================================================\ | Page: 9 | | |==========/ | | Functions: read_password(), get_password() | |=========== | | | | Description: | |============= | | This couple of functions gets the password of our Freebox. There are 4 ways to | | retrieve the password that we try in a decreasing priority order. | | 1- the -p option have already set a password (highest priority) | | 2- it is in the URI in the form http://freebox:myPasswd@10.20.30.40 | | 3- search it in the config file (option -c or default config file) | | 4- ask the user (keyboard input) (lowest priority) | | As main() already read the parameters, and already checked the URI, if 1 or 2 | | these functions will never be called. So here we take care of 3 and 4 only | | | | get_password takes care of 3: trying to find the password in the config files. | | If the option -c was used, the configuration file specified with the option is | | passed to the function who tries to open it. If it fails we warn and continue | | with the default configuration file. | | If one of the opens (specified by -c or default) is sucessful, we search inside | | the file for a line à la bash, containing: | | fbxPassword="MyPasswd" (and no # before that, because # is considered as comment) | | if such a line (uncommented) is found, the password is set, eitherwise we call: | | | | read_password turns out the echo in the console (after saving the settings) then | | reads the password from keyboard. Once done the console's settings are restored. | | | \====================================================================================*/ void read_password( pPassword, maxlen) char *pPassword; int maxlen; { struct termios oldt, newt; int i = 0; int c; /*saving the old settings of STDIN_FILENO and copy settings for resetting*/ tcgetattr( STDIN_FILENO, &oldt); newt = oldt; /*setting the approriate bit in the termios struct*/ newt.c_lflag &= ~(ECHO); /*setting the new bits*/ tcsetattr( STDIN_FILENO, TCSANOW, &newt); /*reading the pPassword from the console*/ while ((c = getchar())!= '\n' && c != EOF && i < maxlen -1){ pPassword[i++] = c; } pPassword[i] = '\0'; printf("\n"); /*resetting our old STDIN_FILENO*/ tcsetattr( STDIN_FILENO, TCSANOW, &oldt); } /*-----------------------------------------------------------------------------------*/ void get_password( pConfigFile ) char *pConfigFile; { /* So... now trying config files, passed with -c first, then default */ FILE *fpConfig=NULL; if (pConfigFile != NULL) { fpConfig=fopen( pConfigFile, "r" ); lprintf( LOG_WARNING, fpConfig==NULL, CONFIG_FILE_NOT_FOUND_WILL_TRY_WITH_DEFAULT_WARN_MSG, pConfigFile ); } if (fpConfig==NULL) { char *HOME; HOME=getenv( "HOME" ); pConfigFile=malloc_or_die( strlen(HOME) + strlen(pDefConfig) + 1 ); strcpy(pConfigFile, HOME); strcpy(pConfigFile + strlen(HOME) , pDefConfig ); fpConfig=fopen( pConfigFile, "r" ); free(pConfigFile); } if ( fpConfig != NULL ) { char aLineBuf[LINE_MAX]; char *pFound; char *pComment; char separator; char *pEndPassword; int passwordLen; while( fgets(aLineBuf, sizeof(aLineBuf), fpConfig) != NULL ) { if ( (pFound=strstr(aLineBuf, aPassVar)) != NULL ) { pComment=strchr(aLineBuf, '#'); if ( pComment==NULL || pComment > pFound ) { pFound+=sizeof(aPassVar)-1; separator=*(pFound++); if ( separator == '"' || separator == '\'' ) { pEndPassword=strchr(pFound, separator); passwordLen= pEndPassword - pFound; pFbxPasswordPost= malloc_or_die( passwordLen + 1 ); memcpy( pFbxPasswordPost, pFound, passwordLen ); *( pFbxPasswordPost + passwordLen ) = '\0'; break; } } } } fclose(fpConfig); } /* No pPassword found so far (could also be that config files existed but no pPassword found inside), so we ask for it */ if (pFbxPasswordPost==NULL) { pFbxPasswordPost=malloc_or_die( (LINE_MAX) ); printf( PASSWORD_PROMPT ); read_password( pFbxPasswordPost, (LINE_MAX) ); } } /*====================================================================================\ | Section | | |==========/ | | | | We finished the introduction, utilities and paramaters reading, now we move to | | network-related initialisations and stuff | | | \====================================================================================*/ /*====================================================================================\ | Page: 10 | | |==========/ | | Functions: curl_easy_setopt_or_die() | |=========== curl_easy_perform_or_die() | | | | Description: | |============= | | These are logging utility functions for respectively curl_easy_setopt and | | curl_easy_perform. | | | | The curl_easy_setopt above will log (and display during init) the option in | | full text with the matrix initialised below. For efficiency, it must be sorted | | by ascending CURLoption. When a new option is used, it SHOULD be added here | | (although non-existing option won't bug). To get the values of the CURLoption | | you can either try to figure then from curl.h, or run the code below: | | | | for(i=0; i< sizeof(aOptCodeTranslator)/sizeof(struct optCodeTranslator); i++) | | printf("(%d)%d=%s\n",i,aOptCodeTranslator[i].option,aOptCodeTranslator[i].pLabel); | | | \====================================================================================*/ static struct optCodeTranslator { CURLoption option; char *pLabel; } aOptCodeTranslator[]= { { /* 41*/ CURLOPT_VERBOSE, "CURLOPT_VERBOSE" }, { /* 80*/ CURLOPT_HTTPGET, "CURLOPT_HTTPGET" }, { /*10001*/ CURLOPT_WRITEDATA, "CURLOPT_WRITEDATA" }, { /*10002*/ CURLOPT_URL, "CURLOPT_URL" }, { /*10010*/ CURLOPT_ERRORBUFFER, "CURLOPT_ERRORBUFFER" }, { /*10015*/ CURLOPT_POSTFIELDS, "CURLOPT_POSTFIELDS" }, { /*10023*/ CURLOPT_HTTPHEADER, "CURLOPT_HTTPHEADER" }, { /*10029*/ CURLOPT_HEADERDATA, "CURLOPT_HEADERDATA" }, { /*10031*/ CURLOPT_COOKIEFILE, "CURLOPT_COOKIEFILE" }, { /*20011*/ CURLOPT_WRITEFUNCTION, "CURLOPT_WRITEFUNCTION" }, { /*20079*/ CURLOPT_HEADERFUNCTION,"CURLOPT_HEADERFUNCTION" }, }; int compare_option( p1, p2 ) const void *p1; const void *p2; { return *((CURLoption *)p1) - ((struct optCodeTranslator *)p2)->option; } CURLcode curl_check_setopt( res, i, option ) CURLcode res; int i; CURLoption option; { if ( res != CURLE_OK ) { struct optCodeTranslator *pOption; char *pLabel; pOption= bsearch( &option, aOptCodeTranslator, sizeof(aOptCodeTranslator)/sizeof(struct optCodeTranslator), sizeof(struct optCodeTranslator), compare_option ); if ( pOption == NULL ) pLabel= UNKNOWN_CURL_OPT_MSG; else pLabel= pOption->pLabel; lprintf( LOG_CRIT, true, CURL_SET_OPT_ERR_MSG, res, pLabel, pReaders[i].curlErrBuf ); } return res; } #define curl_easy_setopt_or_die( i, option, ...) \ curl_check_setopt( curl_easy_setopt( pReaders[ i ].curl, option, __VA_ARGS__), i, option ) CURLcode curl_easy_perform_or_die( iReader ) int iReader; { CURLcode res; res= curl_easy_perform( pReaders[iReader].curl ); if ( pReaders[iReader].pReq==NULL && res== CURLE_WRITE_ERROR ) return CURLE_OK; else { lprintf( LOG_CRIT, res != CURLE_OK, CURL_ERROR_ERR_MSG, res, pReaders[iReader].curlErrBuf ); } return res; } /*====================================================================================\ | Page: 11 | | |==========/ | | Functions: write_trash(), write_alloc(), parse_status_code(), parse_status_range() | |=========== | | | | Description: | |============= | | These are utility and callback functions for curl. | | | | The write functions are callbacks to handle the body of the request. | | write_trash is equivalent to writing the body to /dev/null. So from curl, it is | | always sucessful whatever the body is. It is used during the login where we are | | not interested in the body of the reply, but only by the headers. We still have to | | handle the body, that's what we have this function. | | write_alloc is used to get a body of an unknown size. We allocate memory so that | | we have enough room to copy response from the server. In this case the max | | element of the pbcb is not used because we grow the buffer as much as needed. | | write_copy is used when we already have a buffer to copy the response to. Here | | max is used to ensure we don't copy more data than the buffer side. Typically, this| | is used during read operations where fuse gives us a buffer where we have to put | | the bytes read from the server. | | | | parse_status_code is used, as it's name says, to parse the status code of the | | server's responses. See RFC 2616 for meanings: 200, 404, 403, 206, 416, etc... | | For efficency, you must set the status code do 0 before calling curl_easy_perform | | with this callback for code status. It allows us to avoid scanning all the headers.| | | \====================================================================================*/ size_t write_trash( buffer, size, nmemb, userp) void *buffer; size_t size; size_t nmemb; void *userp; { /* This is the "/dev/null" callback: do nothing with data, allways works! */ return size * nmemb; } size_t write_alloc( buffer, size, nmemb, userp) void *buffer; size_t size; size_t nmemb; pbcb *userp; { size_t l=size * nmemb; userp->pb=realloc_or_die( userp->pb, userp->cb + l); memcpy( (char *)(userp->pb) + userp->cb - 1, buffer, l); userp->cb += l; return l; } /*-----------------------------------------------------------------------------------*/ size_t parse_status_code( responseHeader, size, nmemb, statusCode ) void *responseHeader; size_t size; size_t nmemb; void *statusCode; { if ( *(int *)statusCode== 0 ) { if ( strstr( responseHeader, aHTTPCode ) ) sscanf( (char *)responseHeader + sizeof(aHTTPCode) - 1, "%3d", (int *)statusCode ); } return size * nmemb; } /*-----------------------------------------------------------------------------------*/ size_t parse_status_range( responseHeader, size, nmemb, statusCode ) void *responseHeader; size_t size; size_t nmemb; void *statusCode; { if ( *(int *)statusCode== 0 ) { if ( strstr( responseHeader, aHTTPCode ) ) { sscanf( (char *)responseHeader + sizeof(aHTTPCode) - 1, "%3d", (int *)statusCode ); if ( *(int *)statusCode != HTTPE_PARTIAL_CONTENT ) return -1; } } return size * nmemb; } /*====================================================================================\ | Page: 12 | | |==========/ | | Function: is_login_ok(), server_login(), server_reconnect() | |========== | | | | Description: | |============= | | Logs to the Freebox. | | | | Not much to say, that's just it !.. | | We set the callback for the body (discarding it, we don't need that), and the | | post with the string computed at init. Other important elements of curl has | | already been set at init. | | | | We distinguish a successful and a failed login with the respons satus code. | | When succeeded, we get a 302 (Found) with the location of the main page of the | | admin Web (which we don't care, just wanted the status code!) | | When failed, we get a 200 (OK) and the same page again with the failed message. | | (and again, we discarded that body, just wanted the status code!) | | is_login_ok tests this status code and either fails or returns a CURL code OK. | | | | server_reconnect() is called when we are deamonized (probably!) with fuse. So we use| | the macro to check the curl calls, but we still exit if the password is incorrect. | | If such thing happens, it means something is really wrong, or for instance, the | | Freebox owner changed the password. So there is nothing more we can do than exit. | | | \====================================================================================*/ CURLcode is_login_ok( iReader ) int iReader; { if ( pReaders[iReader].statusCode != HTTPE_FOUND ) { lprintf( LOG_CRIT, true, INCORRECT_PASSWORD_ERR_MSG ); curl_easy_cleanup(pReaders[iReader].curl); exit(EXIT_FAILURE); } return CURLE_OK; } CURLcode server_login( iReader ) int iReader; { pReaders[iReader].statusCode=0; curl_easy_setopt_or_die( iReader, CURLOPT_POSTFIELDS, pFbxPasswordPost ); curl_easy_setopt_or_die( iReader, CURLOPT_WRITEFUNCTION, write_trash); curl_easy_setopt_or_die( iReader, CURLOPT_URL, pFbxLoginURI ); curl_easy_perform_or_die( iReader ); return is_login_ok( iReader ); } /*====================================================================================\ | Page: 13 | | |==========/ | | Functions: get_JSON_num(), copy_JSON_str() | |=========== | | | | Description: | |============= | | These functions are used during the scan of directories to get information out | | of the Json strings. (This is part 1/2 of these utility functions) | | | | get_JSON_num: gets a number from the JSON string, for example: | | ...,"size",12345,"name","foo",.... | | the function receives ',"size",' searches for that and returns the number after | | that. In the example, it would return 12345. | | | | copy_JSON_str: copies a JSON string to the specified buffer. Same as above, the | | function receives the pattern key of the string, ',"name","' in the example | | above. The string is moved to the buffer (which has been allocated enough space) | | unescaping when we copy. The buffer pointer is then moved passed the copied zone, | | ready for the next copy. It is not an error if a pattern is not found (contrary to | | num that always exist), because some nodes don't have mimetypes. | | | \====================================================================================*/ long long get_JSON_num( pJSON, pElt, lElt ) char *pJSON; char *pElt; size_t lElt; { long long ret; pJSON= strstr( pJSON, pElt ); lprintf( LOG_CRIT, pJSON== NULL, ILL_FORMED_JSON_RESPONSE_ERR_MSG, pJSON ); pJSON+=lElt; sscanf( pJSON, "%Ld", &ret); return ret; } void copy_JSON_str( pJSON, pElt, lElt, ppDest ) char *pJSON; char *pElt; size_t lElt; char **ppDest; { int fBackSlash; if ( ( pJSON= strstr(pJSON, pElt)) != NULL ) { pJSON+= lElt; while ( *pJSON!= '"' && *pJSON!= '\0' ) { if ( *pJSON == '\\' ) /* Skipping the \ before escaped characters */ { fBackSlash=1; pJSON++; } else fBackSlash=0; if ( fBackSlash && *pJSON == 'r' ) { *((*ppDest)++)='\r'; pJSON++; } else *((*ppDest)++)= *(pJSON++); } } *((*ppDest)++)='\0'; } /*====================================================================================\ | Page: 14 | | |==========/ | | Functions: len_JSON_str(), get_JSON_type(), compare_nodes() | |=========== | | | | Description: | |============= | | These functions are used during the scan of directories to get information out | | of the Json strings. (This is part 2/2 of these utility functions) | | | | len_JSON_str: is called with the same parameters as copy_JSON_str, except the | | destination buffer, and computes the length of the string. It is called by | | directory scanner before copy_JSON_str to know how much memory we need to allocate | | to fit the strings in. The lenght returned includes the final '\0', and the | | escaped characters are properly handled too. | | | | get_JSON_type: returns the nodeType (see the enum definition at page 2) of the | | current node represented by our JSON string. Simple search (pattern: ',"type","') | | plus compare to known strings values for this field (see constants page 2) | | | | compare_nodes: this one doesn't work with JSON, but simply compare 2 nodes of our | | global tree. It is conformant to the prototype of function expected by qsort. | | nodes are sorted by types first (so dirs before files) then by names. | | | \====================================================================================*/ size_t len_JSON_str( pJSON, pElt, lElt ) char *pJSON; char *pElt; size_t lElt; { char *q; int l=1; /* We start at 1 to alloc space for the final \0 of the string */ if ( ( q=strstr(pJSON, pElt) ) != NULL) /* If pElt is not found, it is not*/ { /* an error, see above. Return 1 for the \0 */ q+= lElt; while ( *q!= '"' && *q!= '\0' ) { if ( *( q++ ) == '\\' ) /* Not counting 2 for escaped characters */ { if ( *q == '"' || *q == '\\' || *q == '/' || *q == 'r') q++; /* Only 3 characters we found escaped are */ /* canaillou2k5: HFS+ on a des fichiers */ /* contenant \r */ else /* ", /, \. So if not that, brk to error */ break; } l++; } if ( *q!= '"' ) /* If not found final '"' = illegal JSON string */ lprintf( LOG_CRIT, true, ILL_FORMED_JSON_RESPONSE_ERR_MSG, pJSON ); } return l; } nodeType get_JSON_type( pJSON ) char *pJSON; { if ( ( pJSON= strstr(pJSON, aType ) ) != NULL ) { pJSON+= sizeof( aType ) - 1; if ( strncmp( pJSON, aFileTypeValueJSON, sizeof(aFileTypeValueJSON)- 1 )== 0 ) return tFILE; if ( strncmp( pJSON, aDirTypeValueJSON, sizeof(aDirTypeValueJSON)- 1 ) == 0 ) return tDIR; if ( strncmp( pJSON, aLinkTypeValueJSON, sizeof(aLinkTypeValueJSON)- 1 )== 0 ) return tLINK; } return tOTHER; } /*-----------------------------------------------------------------------------------*/ int compare_nodes( p1, p2 ) const void *p1; const void *p2; { if ( ((FSNode *)p1)->type < ((FSNode *)p2)->type ) return -1; if ( ((FSNode *)p1)->type > ((FSNode *)p2)->type ) return 1; return strcmp( ((FSNode *)p1)->pName, ((FSNode *)p2)->pName ); } /*====================================================================================\ | Page: 15 | | |==========/ | | Function: store_server_tree() | |========== | | | | Description: | |============= | | This function gets the entire filesystem tree (recursively) from pDir and down | | of our Freebox. | | It is the central function of the initialisation process. | | | | Part 1: we retrieve the directory content (JSON string) from the server (Freebox). | | Very usual stuff here. We need a random for our post. Make the post string with | | our random and the current directory name, give it to curl... perform... | | and don't forget to test thoroughly if all went well. | | | | Part 2: compute the memory needed to store structures and strings for our direc- | | tory. There are a few tricks during this count (tricky JSON!) that are explained | | directly in the code. At the end, we have the memory needed, and the count of | | 'nodes' (items on our directory) and sub-dir. Last thing is the actual allocation. | | | | Part 3: now we populate the structs and copy the strings to the allocated mem. | | Also detailed in the code itself, for example that we allocated memory both for | | FSNode structs and for strings (names, mimetypes) in one single call. | | | | Part 4: we sort nodes of each dir (sub-dirs first then by name) and display a short | | summary so that the user sees it's progressing. The sorting help shorten the time | | for the search algorithm in the fuse operations. We spend a lot of time searching | | for dirs in the paths. As dirs are 1st, ordered, and we know their number, we can | | later use a very efficient bsearch for that. | | | | Part 5: we recurse on child dirs to get the whole tree, as dirs have been sorted | | first in part 4, this is an easy loop! | | | \====================================================================================*/ void store_server_tree( pDir, poParent, nFiles, nDirs, nLinks ) char *pDir; FSNode *poParent; int *nFiles; int *nDirs; int *nLinks; { char *pPostGetDir; char *pDirEscaped; char aRandomNumber[16]; char *p, *q; size_t len=0; nodeType type; int i=0; lprintf(LOG_INFO, true, LIST_DIR_MSG, pDir); pReaders[0].statusCode=0; pReaders[0].pbcb.pb= NULL; pReaders[0].pbcb.cb= 1; /* We sart at 1 to reserve space for the final '\0' */ /*pReaders[0].pbcb.max= ... we don't care for 'max' here as buffer will be allocated */ /*_________________________________________________________________________________*/ /* Part 1: retrieving content of the directory (JSON) from the server (Freebox) */ pDirEscaped= JSONescape( pDir ); snprintf( aRandomNumber, sizeof(aRandomNumber), "%d", rand() ); pPostGetDir=alloc_cat( "{\"jsonrpc\":\"2.0\",\"method\":\"fs.list\",\"id\":", aRandomNumber, ",\"params\":[\"", pDirEscaped, "\",{\"with_attr\":true}]}", NULL ); curl_easy_setopt_or_die( 0, CURLOPT_POSTFIELDS, pPostGetDir ); curl_easy_perform_or_die( 0 ); if ( pDirEscaped != pDir ) free( pDirEscaped ); free( pPostGetDir ); if ( pReaders[0].pbcb.pb != NULL ) { ((char *)(pReaders[0].pbcb.pb))[pReaders[0].pbcb.cb - 1]='\0'; } /* We don't fail just now if the buffer is NULL in case we got a more */ /* relevant message to display with the response status code. */ lprintf( LOG_ERR, pReaders[0].statusCode != 200, HTTP_JSON_ERROR_MSG, pReaders[0].statusCode, (pReaders[0].pbcb.pb == NULL) ? "NULL" : (char *)pReaders[0].pbcb.pb ); /* But now we fail if the buffer is NULL */ lprintf( LOG_ERR, pReaders[0].pbcb.pb == NULL, CURL_NO_DATA_RECEIVED_MSG ); if ( ( p=strstr( pReaders[0].pbcb.pb, aJSONOK )) == NULL ) { lprintf( LOG_WARNING, true, JSON_ERR_ON_DIR_WARN_MSG, (char *)pReaders[0].pbcb.pb ); lprintf( LOG_WARNING, true, JSON_ERR_ON_DIR_CONSEQUENCE_WARN_MSG, pDir ); return; } /*_________________________________________________________________________________*/ /* Part 2: compute the memory needed to store structures and strings */ p=((char *)(pReaders[0].pbcb.pb)) + 1; /* Skip the 1st { that is at the begining */ while( ( q=strchr( p, '{' ) ) != NULL ) { p=strstr( q, aSize ); if ( p==NULL ) /* If we don't find any 'size', it means the */ break; /* dir is empty, because 'size' is ALWAYS */ /* there for each node. Then if so we stop */ /* counting */ p=strchr( p, '}' ); /* We can't search directly for '}' as names */ /* can have this character, after 'size' it's*/ /* safe, because it is the last JSON item, */ /* and it holds a number (so no '}') */ *( p + 1 )='\0'; p+=2; /* In 'log' mode we output the whole JSON */ debug( true, JSON_DUMP_MSG, *nFiles + *nDirs + *nLinks + poParent->detail.child.nNodes, q ); /* Compute space for 'name' and 'mimetype' */ /* the later only if we are a regular file. */ len+= len_JSON_str( q, aName, sizeof(aName)-1 ); type= get_JSON_type( q ); if ( type == tFILE ) len+= len_JSON_str( q, aMime, sizeof(aMime)-1 ); /* Then we count +1 node and +1 dir .... */ /* ... if we are a dir! */ if ( type == tDIR ) poParent->detail.child.nDirs++; poParent->detail.child.nNodes++; } /* Logging all the good work... */ debug( true, NODE_MEMORY_NEEDED_MSG, poParent->detail.child.nNodes, (long)(poParent->detail.child.nNodes * sizeof( FSNode ) + len) ); /* ... and finally allocating that memory! */ poParent->detail.child.pFirstNode= malloc_or_die( poParent->detail.child.nNodes * sizeof( FSNode ) + len ); /*_________________________________________________________________________________*/ /* Part 3: now we populate the structs and copy the strings to the allocated mem. */ /* Above, in one call, we allocated mem both */ /* for FSNode structs and for strings. */ /* q below will point 'after' the structs */ /* thus at the begining of the area where we */ /* will store the strings of each node. */ q=(char *)&poParent->detail.child.pFirstNode[poParent->detail.child.nNodes]; p=pReaders[0].pbcb.pb; for( i= 0; i < poParent->detail.child.nNodes; i++ ) { poParent->detail.child.pFirstNode[i].parent=poParent; poParent->detail.child.pFirstNode[i].pName= q; copy_JSON_str( p, aName, sizeof(aName)-1, &q ); /* Warn about the '?' bug than bugs download */ lprintf( LOG_WARNING, strchr(poParent->detail.child.pFirstNode[i].pName, '?'), BUG_FBX_WARN_MSG, poParent->detail.child.pFirstNode[i].pName ); switch ( poParent->detail.child.pFirstNode[i].type = get_JSON_type( p ) ) { case tFILE: (*nFiles)++; poParent->detail.child.pFirstNode[i].detail.pMimetype= q; copy_JSON_str( p, aMime, sizeof(aMime)-1, &q ); break; case tDIR : (*nDirs)++; poParent->detail.child.pFirstNode[i].detail.child.pFirstNode=NULL; poParent->detail.child.pFirstNode[i].detail.child.nNodes=0; poParent->detail.child.pFirstNode[i].detail.child.nDirs=0; break; case tLINK: /* Nothing can be done but counting links, */ (*nLinks)++; /* see comments in the typedef. */ case tOTHER: /* Nothing (yet!) to do here. */ break; } poParent->detail.child.pFirstNode[i].modification= get_JSON_num( p, aMod, sizeof(aMod) -1 ); poParent->detail.child.pFirstNode[i].size= get_JSON_num( p, aSize, sizeof(aSize)-1 ); /* Same as above, we need to skip the 'name' */ /* because '}' is a valid character in names */ /* Hence the search for 'size' (that is */ /* after 'name' in the order of the JSON) */ p= strchr( strstr( p, aSize ), '}' ) + 2; } free( pReaders[0].pbcb.pb ); /* Free the JSON string we got here, now, all*/ /* have been copied we don't need it anymore.*/ /*_________________________________________________________________________________*/ /* Part 4: we sort nodes of each dir (dirs first then by name) and display count. */ qsort( poParent->detail.child.pFirstNode, poParent->detail.child.nNodes, sizeof( FSNode ), compare_nodes ); /* Display a summary (count of nodes only) */ if ( logLevel == LOG_NOTICE ) /* ... only if we are not at a more detailed */ { /* level than default LOG_NOTICE */ lprintf( LOG_NOTICE, true, TREE_SUMMARY_COUNT_MSG, *nFiles, *nDirs, *nLinks ); fflush( stdout ); /* And flush it, otherwise with no \n we get */ } /* no display and lose the purpose of dis- */ /* playing the summary! */ /* And we log a detailed version of the */ /* sorted directory. */ for( i=0; i < poParent->detail.child.nNodes; i++ ) lprintf( LOG_INFO, true, TREE_DETAIL_MSG, i + *nFiles + *nDirs + *nLinks - poParent->detail.child.nNodes, aTypes[poParent->detail.child.pFirstNode[i].type], poParent->detail.child.pFirstNode[i].size, poParent->detail.child.pFirstNode[i].pName ); /*_________________________________________________________________________________*/ /* Part 5: recurse on child dirs to get the whole tree, as dirs are 1st it's easy. */ for( i= 0; i < poParent->detail.child.nDirs; i++ ) { char *childDir; if ( strcmp( pDir, "/" ) == 0 ) childDir= alloc_cat( pDir, poParent->detail.child.pFirstNode[i].pName, NULL ); else childDir= alloc_cat( pDir, "/", poParent->detail.child.pFirstNode[i].pName, NULL ); store_server_tree( childDir, &poParent->detail.child.pFirstNode[i], nFiles, nDirs, nLinks ); free( childDir ); } } /*====================================================================================\ | Page: 16 | | |==========/ | | Function: free_server_tree() | |========== | | | | Description: | |============= | | Free the memory allocated by the previous function (store_server_tree) | | | | It should be called from the root of the tree to clean the whole tree recursively. | | We test for a NULL pointer passed, as this can occur when we encounter a JSON | | error on a directory, instead of aborting, we just skip that directory. The | | consequence is that the pointer to this inaccessible directory is NULL, and not | | testing it would GP-Fault the loop test. | | | \====================================================================================*/ void free_server_tree( pNode ) FSNode *pNode; { int i; if ( pNode != NULL ) { for (i=0; idetail.child.nDirs; i++) { free_server_tree(pNode->detail.child.pFirstNode + i); } free(pNode->detail.child.pFirstNode); } } /*====================================================================================\ | Page: 17 | Blank page | \====================================================================================*/ /*====================================================================================\ | Page: 18 | Blank page | \====================================================================================*/ /*====================================================================================\ | Page: 19 | Blank page | \====================================================================================*/ /*====================================================================================\ | Section | | |==========/ | | | | We completed the utility, params, and ALL initialisations. Now we move to the | | 'fuse' part of the program, eg the callbacks for fuse operations. | | This next section is 'fuse utils' (utilities for fuse operations) | | | \====================================================================================*/ /*====================================================================================\ | Page: 20 | | |==========/ | | Function: compare_dir(), compare_file(), search_dir() | |========== | | | | Description: | |============= | | compare_dir and compare_file are comparison functions in respect of the type | | needed by bsearch/lsearch. They compare the key passed with the given FSNode. | | As we use length (to avoid putting '\0' in the path) we must do an additional | | test when strncmp says it is equal. It the FSNode is not teminated at the same | | position, it means it is bigger (example: test / test2, strncmp=0 / our test=-1) | | | | search_dir is used to find a directory. It receives a whole absolute path, and | | searches every elements of this path. It returns the pointer to the FSNode | | corresponding to the last element of the path, or NULL if there is at least one | | of the elements in the path that do not exist. | | We don't use tokenize because we are not supposed to put some '\0' inside the | | string coming from fuse. Instead we use index and lengths. | | | \====================================================================================*/ int compare_dir( p1, p2 ) const void *p1; const void *p2; { int res; res= strncmp( ((pbcb *)p1)->pb, ((FSNode *)p2)->pName, ((pbcb *)p1)->cb ); if ( res == 0 && *( ((FSNode *)p2)->pName + ((pbcb *)p1)->cb ) != '\0' ) res= -1; return res; } int compare_file( p1, p2 ) const void *p1; const void *p2; { return strcmp( (char *)p1, ((FSNode *)p2)->pName ); } FSNode * search_dir( pPath ) char * pPath; { FSNode *pNode=&FSRoot; pbcb p; char *q; /* Paranoid test. Normally FUSE is supposed */ if ( *(pPath++) != '/' ) /* to call us with absolute paths only, but */ return NULL; /* we check it anyway. */ if ( *pPath != '\0' ) /* This is a shortcut for root. We could */ /* have skipped the test and let the loop run*/ /* but as FUSE calls us a lot with "/" it is */ /* nice to optimise that */ for( p.pb= pPath; pNode!= NULL; p.pb= q + 1 ) { q=index( p.pb, '/' ); if ( q== NULL ) { p.cb= strlen( p.pb ); if ( p.cb == 0 ) /* Paranoid test again. We shouldn't receive */ break; /* a path with a trailing '/', but just in */ } /* case we do, we will handle it correctly, */ /* & respond as if no trailing '/' was used. */ else p.cb= q - (char *)(p.pb); /* We can 'bsearch' for a directory because */ pNode= bsearch( &p, /* they have been sorted at the init phase. */ pNode->detail.child.pFirstNode, pNode->detail.child.nDirs, sizeof( FSNode ), compare_dir ); if ( q== NULL) /* q is null when we just handled the last */ break; /* element of the path, so we can exit loop */ } return pNode; } /*====================================================================================\ | Page: 21 | | |==========/ | | Function: fill_attr() | |========== | | | | Description: | |============= | | This function fills the 'stat' structure for a given file or directory. | | | | It receives a 'basename' & a directory in the form of a FSNode pointer, and a | | simply fills the 'stat' structure with the relevant information. | | Permission masks are read-only (444) and read+access (554) for directories, as we | | are a read-only filesystem. | | | \====================================================================================*/ int fill_attr( basename, pDir, pStat ) char *basename; FSNode *pDir; struct stat *pStat; { FSNode *thisNode; size_t nNodes; if ( *basename == '\0' ) /* This only happens when we want attrs of */ thisNode=pDir; /* root '/', in this case pDir is the node */ /* we want */ else { /* We have to use lfind and not bsearch */ /* because as we search for both a dir name */ /* or a file name, it is not globally sorted */ nNodes= pDir->detail.child.nNodes; thisNode= lfind( basename, pDir->detail.child.pFirstNode, &nNodes, sizeof( FSNode ), compare_file ); } if ( thisNode == NULL ) return -ENOENT; switch ( thisNode->type ) { case tFILE : pStat->st_mode= S_IFREG | 0444; pStat->st_size= thisNode->size; break; case tDIR : pStat->st_mode= S_IFDIR | 0555; pStat->st_size= thisNode->size; break; case tLINK : pStat->st_mode= S_IFLNK | 0444; pStat->st_size= strlen( basename ); break; case tOTHER: return -ENOENT; } pStat->st_uid= uid; pStat->st_gid= gid; pStat->st_blksize= 4096; pStat->st_atime= pStat->st_mtime= pStat->st_ctime= thisNode->modification; pStat->st_blocks= (( pStat->st_size + 4095 ) / 4096) * 8; return 0; } /*====================================================================================\ | Page: 22 | | |==========/ | | Function: split_path() | |========== | | | | Description: | |============= | | This function splits the path given into a 'basename' and a 'dirname' (in the form | | of FSNode *) | | | | Be aware that it works with pointers of pointers, so that the calling functions | | get the components of the path back to its memory scope. | | Returns -ENOENT when one of the component of the path is not found, or 0 when OK. | | | \====================================================================================*/ static int split_path( pPath, ppBasename, ppDir) char *pPath; char **ppBasename; FSNode **ppDir; { *ppBasename=rindex( pPath, '/' ); if ( *ppBasename== NULL ) /* Paranoid test again (see page 20) as this */ return -ENOENT; /* is not supposed to happen with fuse. */ if ( *ppBasename == pPath ) /* It means our file (pPath) is at the root */ { *ppDir=&FSRoot; (*ppBasename)++; } else { if ( *((*ppBasename) + 1)== '\0')/* This is not supposed to happen either '/' */ return -ENOENT; /* see also comments page 20 */ **ppBasename='\0'; /* We temporarily put a '\0' so that seacrh_ */ /* dir won't GP Fault with unfinished string */ *ppDir= search_dir( pPath ); *((*ppBasename)++)='/'; /* And restoring the '/' here */ if ( *ppDir == NULL ) return -ENOENT; } return 0; } /*====================================================================================\ | Page: 23 | Blank page | \====================================================================================*/ /*====================================================================================\ | Page: 24 | Blank page | \====================================================================================*/ /*====================================================================================\ | Section | | |==========/ | | | | This next section if 'thread utils' (utilities for our multithreaded operations) | | | \====================================================================================*/ /*====================================================================================\ | Page: 25 | | |==========/ | | Function: pthread_mutex_lock_or_die(), pthread_mutex_unlock_or_die() | |========== sem_init_or_die(), sem_wait_or_die(), sem_post_or_die() | | | | Description: | |============= | | These macros ensure the returen code is properly tested for mutex ans semaphore | | operations. | | | | As these are critical for multithreaded programs, if something goes wrong, we log | | and exit. | | | \====================================================================================*/ #define pthread_mutex_lock_or_die( pMutex ) \ lprintf( LOG_CRIT,\ pthread_mutex_lock( pMutex ),\ MUTEX_LOCK_FAILED_ERR_MSG\ ) #define pthread_mutex_unlock_or_die( pMutex ) \ lprintf( LOG_CRIT,\ pthread_mutex_unlock( pMutex ),\ MUTEX_UNLOCK_FAILED_ERR_MSG\ ) #define sem_init_or_die( pSem, pShared, value ) \ lprintf( LOG_CRIT,\ sem_init( pSem, pShared, value ),\ SEM_INIT_FAILED_ERR_MSG\ ) #define sem_wait_or_die( pSem ) \ lprintf( LOG_CRIT,\ sem_wait( pSem ),\ SEM_WAIT_FAILED_ERR_MSG\ ) #define sem_post_or_die( pSem ) \ lprintf( LOG_CRIT,\ sem_post( pSem ),\ SEM_POST_FAILED_ERR_MSG\ ) /*====================================================================================\ | Page: 26 | | |==========/ | | Function: AllocElement(), FreeElement(), DestroyElements() | |========== | | | | Description: | |============= | | Functions working on generic chained FILO simple lists (pools of resources) multi- | | thread safe (with locks). | | | | AllocElement returns an element from the FILO list if any, eitherwise allocates a | | new element and returns it. On allocation of a new element, the user function is | | called (if the memory allocation worked) so that further initialisations can be | | done. | | FreeElement returns the element to the FILO list. | | | | To work on any generic list, the structures on which these functions work MUST | | start with the pointer to the pointer to the next element of the list. The rest of | | the structure is totally free. | | | | 1st two functions need the 'head' (address of the pointer on 1st element) of the | | list, the address MUST NOT be NULL (the pointer at the address can be NULL: when | | the list is empty), and the lock pointer (can be NULL if we don't want a lock). | | Alloc must also be given the size of our structure (must be greater than size of a | | of pointer as we must have a pointer at the start of the structure, plus other | | elements eitherwise it's an useless list of pointers!) and the user function (can | | be NULL). The user function receives an unique parameter: pointer on successfuly | | allocated new element, and must return an int: either 0 for success, or anything | | else for error. On error, the element is freed and AllocElement will return NULL. | | AllocElement returns a pointer on an element (if successfully retrieved from the | | list or allocated) or NULL on error (bad param, alloc failed, lock failed....) | | FreeElements must be given the pointer to free in addition with the head of list. | | It returns the element that was 'freed' (returned to the 'pool' -list) on success | | or NULL on error. Error occurs when incorrect params are passed of if lock fails. | | | | DestroyElements receives a pointer to the head of the list (unlike the other func.) | | and the callback. Its role is to free the elements recursively from the element | | passed as 1st param, to the end of the list. For each element, the callback is | | called with the element so that the required 'cleaning' action can be taken, then | | if no error, the element is freed. If all is ok, return is NULL, eitherwise the | | pointer to the first element that could not be freed is returned. This happens only| | when the user callbaks returns non-zero (indicating an error). | | | \====================================================================================*/ void * AllocElement( ppFirstElement, sizeElement, pLock, initfct ) void **ppFirstElement; size_t sizeElement; pthread_mutex_t *pLock; int initfct(void *); { void *pElement=NULL; if ( ppFirstElement != NULL && /* These 2 conditions are errors, they should*/ sizeElement > sizeof( void *) /* not happen, but we test to be safe */ ) { if (pLock != NULL) pthread_mutex_lock_or_die( pLock ); /*Protect so that only 1 thread allocates*/ if ( *ppFirstElement == NULL ) { pElement= malloc_or_die( sizeElement ); if ( initfct != NULL ) if ( initfct(pElement) != 0 ) { free(pElement); pElement=NULL; } } else { pElement= *ppFirstElement; *ppFirstElement= *(void **)pElement; } if (pLock != NULL) pthread_mutex_unlock_or_die( pLock ); /* All done! Unlock for next list alloc*/ /* We don't test here as there is nothing */ } /* good we can do if unlock does not work! */ return pElement; } void * FreeElement( ppFirstElement, pElement, pLock ) void **ppFirstElement; pthread_mutex_t *pLock; void *pElement; { void *pRet= NULL; if ( ppFirstElement != NULL && /* These 2 conditions are errors, they should*/ pElement != NULL /* not happen, but we test to be safe */ ) { if (pLock != NULL) pthread_mutex_lock_or_die( pLock ); /* Protect so that only 1 thread frees */ *(void **)pElement= *ppFirstElement; *ppFirstElement= pRet = pElement; if (pLock != NULL) pthread_mutex_unlock_or_die( pLock ); /* All done! Unlock for next list alloc*/ } return pRet; } void * DestroyElements( pFirstElement, termfct ) void *pFirstElement; int termfct(void *); { void *pElement; if ( pFirstElement != NULL ) if ( (pElement= DestroyElements( *(void **)pFirstElement, termfct )) != NULL ) return pElement; if ( termfct != NULL ) if ( termfct( pFirstElement ) != 0 ) return pFirstElement; free( pFirstElement ); return NULL; } /*====================================================================================\ | Page: 27 | | |==========/ | | Function: AllocFS() | |========== | | | | Description: | |============= | | Callbacks for functions on previous page. | | | | AllocFS is simply the user functions used when allocating 'files' (the structure | | used to track an opened file during its transfer). | | We don't need a callback to clean FS structures, at they are just memory, and we | | don't log this level of detail. | | | \====================================================================================*/ int AllocFS( pFS ) fs_t *pFS; { pFS->iFS= ++nFS; lprintf( LOG_INFO, true, NEW_FILE_ALLOCATED_MSG, pFS->iFS); return 0; } /*====================================================================================\ | Page: 28 | | |==========/ | | Function: Respond(), store_data(), fill_request() | |========== | | | | Description: | |============= | | Buffer filling and response to caller. | | | | Respond (as we can guess!) simply sends the buffer back from our reader to the | | caller (which in this case is a fuse callback itself). | | | | store_data fills the pbcb (param 1) with the buffer and lenght given as params 2&3. | | It takes care of the max lengths given by pbcb, and moves the cb according to | | the number of bytes copied. The number of bytes copied is returned (it can be less| | than cb if there is not enough 'room' left in our pbcb to dump all the 'buffer') | | | | fill_request calls store data, and then respond to the request on the double | | condition that: (obviously!) the pbcb is 'full', and also that we have some bytes | | left from the 'buffer'. If all the buffer was 'consumed', we don't respond | | because it could mean we are going back to the main loop of the reader (finished | | reading) that wil respond itself. | | | \====================================================================================*/ void Respond( pReq ) request_t *pReq; { debug( true, RESPOND_READ_MSG, pReq->pFS->iReader, pReq->pFS->iFS, pReq->error, pReq->sizeRead ); sem_post_or_die(pReq->pSemResp ); } size_t store_data( pPbcb, buffer, cb) pbcb *pPbcb; void *buffer; size_t cb; { size_t l= (cb <= (pPbcb->max - pPbcb->cb) ) ? cb : pPbcb->max - pPbcb->cb; memcpy( ((char *)pPbcb->pb) + pPbcb->cb, buffer, l); pPbcb->cb+= l; return l; } size_t fill_request( pReader, buffer, cb) reader_t *pReader; void *buffer; size_t cb; { size_t l= store_data( &pReader->pReq->pbcb, buffer, cb); if ( l < cb && pReader->pReq->pbcb.cb == pReader->pReq->pbcb.max ) { pReader->pReq->error= 0; pReader->pReq->sizeRead= pReader->pReq->pbcb.max; Respond( pReader->pReq ); pReader->pReq= NULL; } return l; } /*====================================================================================\ | Page: 29 | Blank page | \====================================================================================*/ /*====================================================================================\ | Section | | |==========/ | | | | And now, after the 'fuse utils' we have the real stuff: 'fuse callbacks' section. | | | \====================================================================================*/ /*====================================================================================\ | Page: 30 | | |==========/ | | Function: fbx_getattr_wrapper(), fbx_getattr() | |========== | | | | Description: | |============= | | Fuse callback for getattr (wasn't it obvious!) | | | | It must fill the 'stat' structure provided with stat informations of the file/dir/ | | link provided in path. | | As we already have the utility functions to split the path (split_path) and fill | | the attr struc (fill_attr) we just need call those 2 functions! | | | | As explained in the generic topic for these functions, the wrapper just adds a | | logging including the return code of the actual function. | | | \====================================================================================*/ static int fbx_getattr( pPath, pStat ) const char *pPath; struct stat *pStat; { char *pBasename; FSNode *pDir; int res; res= split_path( pPath, &pBasename, &pDir ); if ( res != 0 ) return res; return fill_attr( pBasename, pDir, pStat ); } static int fbx_getattr_wrapper( pPath, pStat ) const char *pPath; struct stat *pStat; { int retCode; retCode=fbx_getattr( pPath, pStat ); debug( true, FUSE_GETATTR_MSG, pPath, retCode); return retCode; } /*====================================================================================\ | Page: 31 | | |==========/ | | Function: fbx_readdir_wrapper(), fbx_readdir() | |========== | | | | Description: | |============= | | Fuse callback for readdir (wasn't it obvious too!) | | | | This function receives a pPath that MUST be a dir and returns the content of it | | through calls to 'filler' function. | | There are 2 mode, with and without the use of offset (see fuse doc). As our most | | time consuming part is to execute the search_dir (can involve several bsearch) | | it would be counter productive to try to 'optimise' using the 'return offset' | | mode. This would require more calls to search_dir and be more CPU intensive. | | Whereas once we have found the directory, all the rest is just moving bytes from | | the pointed structures through the filler function. | | Optmization: we don't put the stat in the filler! Looked as a good idea, but if we | | do so, when the user does for example a ls on a directory, the underlying fuse will| | repetitively call readdir to get the attributes of each files in the directory. If | | we call filler with "NULL", it will call readdir once, then getattr for each file | | which costs less because we don't need to loop on the directory, get all attrs and | | call filler so many times! It does also makes the code simpler! | | | \====================================================================================*/ static int fbx_readdir( pPath, pBuf, filler) const char *pPath; void *pBuf; fuse_fill_dir_t filler; { FSNode *pDir; int i; pDir=search_dir( pPath ); if ( pDir == NULL) return -ENOENT; if ( filler(pBuf, ".", NULL, 0) ) return -ENOENT; if ( filler(pBuf, "..", NULL, 0) ) return -ENOENT; for ( i=0; i< pDir->detail.child.nNodes; i++ ) { if ( filler(pBuf, pDir->detail.child.pFirstNode[i].pName, NULL, 0) ) return -ENOENT; } return 0; } static int fbx_readdir_wrapper( pPath, pBuf, filler, offset, pfi ) const char *pPath; void *pBuf; fuse_fill_dir_t filler; off_t offset; struct fuse_file_info *pfi; { (void) offset; (void) pfi; int retCode; retCode=fbx_readdir( pPath, pBuf, filler ); debug( true, FUSE_READDIR_MSG, pPath, offset, retCode); return retCode; } /*====================================================================================\ | Page: 32 | | |==========/ | | Function: fbx_open_wrapper(), fbx_open() | |========== | | | | Description: | |============= | | Fuse callback for open (wasn't it obvious again!) | | | | We don't really need to 'open' the file here. We are just checking that the file | | does exist and that the open mode is correct (still being paranoid). | | To save time during read and avoid having to scan the path, we save the pointer | | to the FSNode of the file (we had to compute it to check file existence) in the | | user structure as the fh (file handle). Thus susbsequent read can use this handle | | (in fact a FSNode *) directly, with no need to scan and check the path. | | Note that we have to save the fh as (unsigned long) although it is a pointer, | | because of the type of this member of the struct. Read will cast it back to ptr. | | | \====================================================================================*/ static int fbx_open(const char *pPath, struct fuse_file_info *fi) { char *pBasename; FSNode *pDir; FSNode *pNode; int res; size_t nNodes; fs_t *pFS; res= split_path( pPath, &pBasename, &pDir ); if ( res != 0 ) return res; nNodes= pDir->detail.child.nNodes; pNode = lfind( pBasename, pDir->detail.child.pFirstNode, &nNodes, sizeof( FSNode ), compare_file ); if ( pNode == NULL ) return -ENOENT; if ( (fi->flags & 3) != O_RDONLY) /* Paranoid -SHOULD NOT happen- because file */ return -EACCES; /* attributes are all 444 and FS has been */ /* given the -o ro (read-only) option */ if ( pNode->type== tDIR ) /* Paranoid -SHOULD NOT happen because FS */ return -EACCES; /* should not attempt an 'open' on a dir! */ pFS= AllocElement( &pFirstFS, sizeof(fs_t), &lAlloc, AllocFS ); if (pFS == NULL) return -ENOENT; pFS->iReader= -1; pFS->pNode= pNode; fi->fh=(unsigned long)pFS; return 0; } static int fbx_open_wrapper( pPath, fi ) const char *pPath; struct fuse_file_info *fi; { int retCode; retCode=fbx_open( pPath, fi ); lprintf( LOG_INFO, true, FUSE_OPEN_MSG, (retCode==0) ? ((fs_t *) (uintptr_t) fi->fh)->iFS : -1, pPath, retCode ); return retCode; } /*====================================================================================\ | Page: 33 | | |==========/ | | Function: fbx_release_wrapper(), fbx_release() | |========== | | | | Description: | |============= | | Fuse callback for release (=close) of files. | | | | We can now release the file handle we acquired during the 'open'. If the handle | | has been used for reads (means a reader has been allocated to that handle) we | | decrement the count of files on that reader. We do also clean cached bytes (if any) | | on this file handle. | | At last, we call FreeElement to release our file handle. | | | \====================================================================================*/ static int fbx_release( pFS ) fs_t *pFS; { int iReader; iReader= pFS->iReader; if ( iReader != -1 ) { pthread_mutex_lock_or_die( &pReaders[iReader].lThis ); { pReaders[iReader].nFiles--; if (pReaders[iReader].pFS == pFS) { pReaders[iReader].pFS = NULL; if ( pReaders[iReader].pbcb.cb != 0 ) { debug( true, BUFFER_CLEANUP_AT_RELEASE_MSG, pReaders[iReader].pbcb.cb, pReaders[iReader].offset, pFS->pNode->pName ); pReaders[iReader].stats.bytesLost += pReaders[iReader].pbcb.cb; } } } pthread_mutex_unlock_or_die( &pReaders[iReader].lThis ); } if ( FreeElement( &pFirstFS, pFS, &lAlloc ) == pFS ) return 0; else return -1; } static int fbx_release_wrapper( pPath, fi ) const char *pPath; struct fuse_file_info *fi; { int retCode; fs_t *pFS; int iFS; if ( fi->fh == 0 ) return -ENOENT; pFS= (fs_t *) (uintptr_t) fi->fh; iFS= pFS->iFS; retCode=fbx_release( pFS ); lprintf( LOG_INFO, true, FUSE_RELEASE_MSG, iFS, retCode); return retCode; } /*====================================================================================\ | Page: 34 | | |==========/ | | Function: //Prototypes of functions// | |========== | | | | Description: | |============= | | Prototypes of the necessary asynchronous functions, so that we can have a source | | in a more logical order, grouping here all fuse callbacks. | | | \====================================================================================*/ void * async_reader(void *pUserData ); void init_reader (int iReader, bool fInitAll ); void start_reader(int iReader ); void log_stats (int level, stat_t *pStats ); void end_reader (int iReader, stat_t *pStats ); /*====================================================================================\ | Page: 35 | | |==========/ | | Function: fbx_destroy_wrapper(), fbx_destroy() | |========== | | | | Description: | |============= | | These functions are called upon fuse exit. | | | | This can happen in the "normal" situation where the user does a fusermount -u | | and also when whe exit or abort (fuse_main sets trap for that). | | | | What we do here is call the 'logout' facility to invalidated the cookie. It is | | a 'security' feature to avoid further reuse of our cookie. At the moment it is | | another 'paranoid' feature because the 'no-security-at-all' feature that Free | | implemented give an attacker plenty of other ways to gain access to our Freebox. | | But even so, it was not an excuse to be 'not-secured' ourselves. | | | \====================================================================================*/ void fbx_destroy_wrapper( void *userData ) { (void)userData; int i; stat_t stats= { 0, 0, 0, 0, 0, 0 }; for ( i= 0; i< maxReaders; i++ ) end_reader( i, &stats ); lprintf( LOG_NOTICE, true, STAT_TOTAL_MSG ); log_stats( LOG_NOTICE, &stats ); closelog(); free( pThreadRead ); free( pReaders ); DestroyElements( pFirstFS, NULL ); } /*====================================================================================\ | Page: 36 | | |==========/ | | Function: fbx_init() | |========== | | | | Description: | |============= | | Initialisations that must be done once we are a daemon. | | | | As fuse is normally (unless options like -f, -d) a daemon, some initialisations must| | be deferred to this phase. For this purpose, the first thing fuse does is to calls| | us on this 'init' function. | | | | So here we do basic things for a daemon: | | - open the daemon syslog (if not in foreground) | | - allocates the Thread buffer to store the tid when we launch the readers threads. | | - and fully initialize re-initialise reader 0 (we have already used if for the | | reading of the Freebox tree, and we don't allocate readers in advance see comment| | at the start of next section) | | | | Nothing else happens as long as we don't return to fuse (eg. as long as our init is | | not complete fuse won't call any other callback). Nevertheless we must lock the call| | to start the reader (see comment in the start_reader function) | | | \====================================================================================*/ void * fbx_init( conn ) struct fuse_conn_info *conn; { (void)conn; if ( !fForeground ) openlog( PROG_NAME, 0, LOG_DAEMON); lprintf( LOG_NOTICE, true, DAEMON_BACKGROUNG_INIT_MSG ); pThreadRead= malloc_or_die( sizeof(pthread_t) * maxReaders ); pthread_mutex_lock_or_die( &pReaders[0].lThis ); { init_reader( 0, true ); start_reader( 0 ); } pthread_mutex_unlock_or_die( &pReaders[0].lThis ); return NULL; } /*====================================================================================\ | Section | | |==========/ | | | | Now comes the 'fuse-read' section with the asynchronous bits and pieces. | | | | Architecture: | |============== | | The goal here, is to be able to respond to a read request (coming through the fuse | | callback) and keep-on reading when it can be of some benefit (read-ahead). | | To do so, we need to have separate threads (from the fuse threads) that do the | | reading. And we need a way to speak back and forth from the fuse threads where | | the read 'requests' come, to the 'readers'. | | Thus the standard path of a request would be: | | >1)- (User application => read => kernel ) | | => 2)- (kernel => fuse-in-kernel) | | => 3)- (fuse-in-kernel => fuse-libraries-in-user-space) | | => 4)- (fuse-libraries-in-user-space) ==> our-fuse-read-callback | | => 5) our-fuse-read-callback ==dispatch-to-a=> reader-thread | | => 6) reader-thread ==calls-curl==> curl-callbacks | | => 7) Here we are on the wire to the Freebox | | <= 8) and data flow 'down' to our curl-callbacks | | <= 9) read-thread <==gets-it data-from== curl-callbacks | | <= 10) our-fuse-read-callback <==response-by== read-thread (on buf. full) | | <= 11) (fuse-libraries-in-user-space) <==response-by== our-fuse-read-callback| | <= 12)- (fuse-in-kernel <= fuse-libraries-in-user-space) | | <= 13)- (kernel <= fuse-in-kernel) | | <14)- (User application <= response to read <= kernel) | | | | Of course, a lots of that (what is in parenthesis) happens outside of our program. | | We come in the play from the fuse-callbacks to the moment we answer to that. | | The first desgin did exactly this path for all requests (see first page comments). | | | | The asynchronous design can do better when it detects possible optimisations. | | For instance, if we detect a sequential read pattern, instead of triggering a read| | to the server for each incoming request, we issue a single read from the position | | we detected the 'sequential behaviour' to the end of the file. Thus at step 10, | | when the buffer for the current request is full, we respond to the fuse callback | | but continue reading (which is possible thanks to the fact we are in another | | thread). Hopefully we already have a second sequential request to fill, and if we | | don't we write to our internal buffer which give time to the 'continuation' | | request to arrive. When it arrives, we move the date from the internal buffer to | | the request, and continue reading it. | | In fact, a lot of Linux programs feed correctly our program with continuous requests| | cp, rsync, Nautilus copy, etc... rsync even gives us 2 requests at a time. | | Also note that to help this situation happen, we run at a slightly lower priority, | | so that the user application gets more chance to give us a continuation request | | quickly (as it has more priority than us). | | | | There are 3 read patterns used to optimise: | | -Sequential: this is when requests arrive fast enough so that we don't need to stop | | reading. The latest moment we expect a continuation request is when our internal | | buffer is full. If such thing happens, we move to a second pattern which is | | -Slow (sequential): it is a pattern where requests are still sequential, but arrive | | slower than the reading from the server. This is typically used when streaming. | | For example, when you stream a 256Kbps MP3, even on a 1Mbps upload ADSL, you are | | in the 'slow' pattern. When this pattern is detected and the buffer is full, to | | avoid aborted reads which are 'expensive', we issue 'internal requests' as soon | | as the internal buffer is empty. This ensure the streming application has better | | chance to get its data as soon it needs it. | | -Random: this is the default pattern when none of the above is detected. It falls | | back to the algorithm used by the first basic design, although with thread | | switches, but some minor 'optimisations' are added. | | | | Operation flow: | |=============== | | When a file is opened (open-callback) it is given a 'FS' handle taken from a pool | | of handle. At this stage, the handle is not linked to any reader thread. | | When a read occurs on this previously opened file, the read-callback builds | | a 'request' from its stack. The request is initialised, notably with the incoming | | buffer/max-length from fuse, and with a semaphore (taken from the pool of | | 'exchanges' -CTOSian vocabulary!-). | | If it was the 1st read request on this 'FS' handle, we associate it with a reader | | (see fbx_read_wrapper for the algo). | | Then the request is sent to the queue of that reader. | | At this point there are already powerful optimisations when we have data in our | | internal buffer. Typically on 'Slow' mode, we almost all the time respond directly| | from the data we already have. | | At this point we have triggered the Async_Reader. On its main event loop, it takes | | the request and analyses to 'guess' the read pattern used by the caller (or if | | we need to alter the previouly guessed pattern). Then is does an actual read via | | a curl call. The read always starts at the begining of the buffer (plus cached | | data if any) and spans accordingly to the detected read pattern. | | We are then 'awaken' by curl on our callback that would fill the buffer. Once full | | either we return to the main even loop of our reader if we are in 'random' pattern| | or we respond directly from the curl-callback to the fuse-callback, in order to | | continue reading (next request or internal buffer). | | When the read request comes back to the fuse callback (either ways) the 'exchange' | | is freed to the pool then we respond to fuse. | | And finally, when the file is closed (release for fuse) we free the 'FS' handle to | | the pool of handles. | | | \====================================================================================*/ /*====================================================================================\ | Page: 40 | | |==========/ | | Function: server_logout(), log_stats(), end_reader() | |========== | | | | Description: | |============= | | This ends the reader, writes out the stats for that reader, and do the cleanup. | | | | server_logout issues a logout request to the Freebox. The purpose is to invalidate | | the cookie so that no further request could be done reusing that cookie. It is | | a 'security' feature, although at the moment, with the no-security policy existing| | with the remote admin, it is quite useless as an evesdropper could much more | | easyly use the password that flow unencrypted. | | If there is an error in this request (perform) we return it. At this stage it | | is not fatal because anyway we are exiting the program. (Could be for example that| | the cookie is already invalid because we try to logout on a reader where that | | has been idle for too long) | | | | log_stats logs the read statistics (for individual readers and global stats). | | Individual reader stats are displayed at 'info' level, and global at 'notice'. | | | | end_reader does the overall cleaning job with these steps: | | -1) logout from the server (Freebox) | | -2) stop the reader and destroys ('joining') the thread. | | -3) log stats and add them to the global counter | | -4) objects (curl handle, semaphore) and memory cleanup | | | \====================================================================================*/ CURLcode server_logout( iReader ) int iReader; { pReaders[iReader].statusCode=0; curl_easy_setopt_or_die( iReader, CURLOPT_HTTPGET, 1 ); curl_easy_setopt_or_die( iReader, CURLOPT_HTTPHEADER, NULL ); curl_easy_setopt_or_die( iReader, CURLOPT_HEADERFUNCTION, parse_status_code ); curl_easy_setopt_or_die( iReader, CURLOPT_HEADERDATA, &pReaders[iReader].statusCode); curl_easy_setopt_or_die( iReader, CURLOPT_WRITEFUNCTION, write_trash ); curl_easy_setopt_or_die( iReader, CURLOPT_URL, pFbxLogoutURI ); return curl_easy_perform( pReaders[iReader].curl ); } void log_stats( level, pStats ) int level; stat_t *pStats; { lprintf( level, true, STAT_BYTES_READ_MSG , pStats->bytesRead ); lprintf( level, true, STAT_BYTES_CACHED_MSG , pStats->bytesCached ); lprintf( level, true, STAT_BYTES_LOST_MSG , pStats->bytesLost ); lprintf( level, true, STAT_FILES_READ_MSG , pStats->nFS ); lprintf( level, true, STAT_READS_COUNT_MSG , pStats->nReads ); lprintf( level, true, STAT_ABORTED_READS_MSG, pStats->nAborts ); } /*-----------------------------------------------------------------------------------*/ void end_reader( iReader, pStats ) int iReader; stat_t *pStats; { if ( pReaders[iReader].pSemPutReq != NULL ) { /*_____________________________________________________________________________*/ /* Part 1: logout from the server (Freebox) */ if ( server_logout( iReader ) == CURLE_OK && pReaders[iReader].statusCode== HTTPE_OK ) lprintf( LOG_NOTICE, true, DISCONNECT_READER_SUCCESS_MSG, iReader); else lprintf( LOG_WARNING, true, DECONNEXION_FAILED_WARN_MSG, iReader, pReaders[iReader].statusCode ); /*_____________________________________________________________________________*/ /* Part 2: stopping the readers and 'joining' the thread */ pReaders[iReader].command |= COMMAND_STOP; sem_post_or_die(pReaders[iReader].pSemPutReq ); if ( pthread_join( *(pThreadRead + iReader), NULL ) ) lprintf( LOG_ERR, true, READ_THREAD_STOP_FAILED_ERR_MSG , iReader ); else lprintf( LOG_INFO, true, READ_THREAD_STOPPED_MSG, iReader ); /*_____________________________________________________________________________*/ /* Part 3: log stats and add themn to the global counters */ lprintf( LOG_INFO, true, READER_STATS_DETAIL_MSG, iReader ); log_stats( LOG_INFO, &pReaders[iReader].stats ); pStats->bytesRead += pReaders[iReader].stats.bytesRead ; pStats->bytesCached += pReaders[iReader].stats.bytesCached ; pStats->bytesLost += pReaders[iReader].stats.bytesLost ; pStats->nFS += pReaders[iReader].stats.nFS ; pStats->nReads += pReaders[iReader].stats.nReads ; pStats->nAborts += pReaders[iReader].stats.nAborts ; /*_____________________________________________________________________________*/ /* Part 4: objects (curl handle, semaphore) and memory cleanup */ curl_easy_cleanup( pReaders[iReader].curl ); lprintf( LOG_ERR, sem_destroy( pReaders[iReader].pSemPutReq ), SEM_DESTROY_ERR_MSG ); free( pReaders[iReader].pSemPutReq ); } } /*====================================================================================\ | Page: 41 | | |==========/ | | Function: fbx_read_wrapper() | |========== | | | | Description: | |============= | | Fuse callback for read requests. | | | | Unlike other callbacks, this one does (usually) do the job by itself. When a read | | is necessary, it is submitted to a readers. On some exceptions ('empty' read, read | | fully cached) the callback answers directly as a 'shortcut' to avoid thread switch.| | | | -1) We do some 'paranoid' tests, and initialize our 'Request' (to send it later | | to the selected reader). | | -2) If the is the 1st read for this file, we select the 'best' reader possible. | | The algorithm for chosing a reader will select: | | - an idle or uninitialized reader (if any) | | - otherwise, a reader NOT doing sequential read (if any) | | - When a non-idle/uninit. reader is selected, we take the one with less files | | on it. | | -3) Now we have a reader (was already there, or we selected it at 2), so we look | | if we have bytes in our 'internal buffer' (the read-ahead cache) and if they | | correspond to the current incoming request. If so, we move as many bytes as | | possible in the incoming read, then move the remaining (if any) cached bytes | | at the begining of the internal buffer and adjust indexes and lengths. | | -4) So here, it might be possible (and mostly probable in 'Slow') mode, that our | | read is 100% full as we might already have copied all the necessary data from | | our internal read-ahead cache. If so we can directly respond to the read. | | But before we respond we see if we must do another request to fill the internal | | buffer. We do so in Slow mode, when the internal buffer is empty, we have no | | other pending request in the queue, and not already requested the read-ahead. | | -5) When the incoming request is not 100% 'full' (no cache-hit or partial hit) we | | queue on the reader's waiting list. While doing so, we check if there might be | | some clever 'inversions' to do, so that we do, as much as possible, sequential | | reads. As some applications (rsync for example) use several reads in parallel | | this situation happens. Once the request is queued, we post on the semaphore | | to indicate to the reader that it has something to do. If the async reader was | | waiting on it's main loop, the 'sem_post' will awake it. Before releasing the | | mutex on that reader, we make sure to complete our request with an 'exchange' | | so that the reader can post back the response. | | -6) And now, we just need to wait for the reader to do its job and respond to our | | request. Once it is done, we can free resources, and reply to fuse. | | | \====================================================================================*/ static int fbx_read_wrapper( pPath, buf, size, offset, fi ) const char *pPath; char *buf; size_t size; off_t offset; struct fuse_file_info *fi; { off_t end; fs_t *pFS; request_t req; static __thread sem_t semResp; static __thread bool fSemOnce= true; if (fSemOnce) { fSemOnce= false; sem_init_or_die( &semResp, 0, 0 ); } /*_________________________________________________________________________________*/ /* Part 1: Paranoid tests, and The Request initialisation. */ /* Paranoid for fi->fh as we should not be */ /* here because we would have errored on open*/ /* pPostReadRoot NULL when the root of our */ /* mount has a '?' (bug Freebox we can't read*/ if (pPostReadRoot == NULL || fi->fh == 0) return -ENOENT; pFS= (fs_t *) (uintptr_t) fi->fh; if ( size== 0 || offset >= pFS->pNode->size ) return 0; /* Not an error, when we try to read at or */ /* past the last byte, or try to read 0 */ /* bytes. We just return that there was */ /* nothing to read (eg 0 byte) */ end= offset + size; if ( end > pFS->pNode->size ) end= pFS->pNode->size; req.pbcb.pb= buf; req.pbcb.cb= 0; req.pbcb.max= end - offset; req.offset= offset; req.pFS= pFS; req.pPath= (char *)pPath; /*_________________________________________________________________________________*/ /* Part 2: Selection of a reader when it is the 1st read for this file. */ if (pFS->iReader == -1) /* No need locking as this is a semi constant*/ { /* it is only changed on the 1st read of file*/ int i, iMin, minFiles; /* and when the 'handle' is re-used by open */ bool fSEQ; for ( i= iMin= minFiles= 0, fSEQ=true; i < maxReaders; i++) { pthread_mutex_lock_or_die( &pReaders[i].lThis ); { if ( pReaders[i].nFiles == 0 ) { pReaders[i].nFiles= 1; pReaders[i].readType= (fNoCache) ? rRAND : rINIT; if ( pReaders[i].pSemPutReq== NULL ) { init_reader( i, true ); start_reader( i ); } pthread_mutex_unlock_or_die( &pReaders[i].lThis ); minFiles= 0; iMin= i; break; } else if ( ( minFiles== 0 ) || ( fSEQ && pReaders[i].readType != rSEQ ) || ( pReaders[i].nFiles < minFiles && ( ( pReaders[i].readType == rSEQ ) == fSEQ ) ) ) { minFiles= pReaders[i].nFiles; fSEQ = (pReaders[i].readType == rSEQ); iMin = i; } } pthread_mutex_unlock_or_die( &pReaders[i].lThis ); } /* At the point, iMin holds the reader compu-*/ pFS->iReader= iMin; /* ted by the selection algorithm (see */ pReaders[iMin].stats.nFS++; /* comments in the page header) */ if ( minFiles == 0 ) lprintf( LOG_INFO, true, FILE_ASSOCIATED_IDLE_READER_MSG, pFS->iFS, iMin ); else { pthread_mutex_lock_or_die( &pReaders[iMin].lThis ); { pReaders[iMin].nFiles++; } pthread_mutex_unlock_or_die( &pReaders[iMin].lThis ); lprintf( LOG_INFO, true, FILE_ASSOCIATED_BUSY_READER_MSG, pFS->iFS, iMin ); } } /*_________________________________________________________________________________*/ /* Part 3: fill the incoming buffer with the 'cache' if it fits. */ pthread_mutex_lock_or_die( &pReaders[pFS->iReader].lThis ); { if ( pReaders[pFS->iReader].pbcb.cb != 0 && req.pFS == pReaders[pFS->iReader].pFS && req.offset == pReaders[pFS->iReader].offset ) { size_t l= store_data( &req.pbcb, pReaders[pFS->iReader].pbcb.pb, pReaders[pFS->iReader].pbcb.cb ); pReaders[pFS->iReader].pbcb.cb -= l; pReaders[pFS->iReader].offset += l; memmove( pReaders[pFS->iReader].pbcb.pb, ((char *)pReaders[pFS->iReader].pbcb.pb) + l, pReaders[pFS->iReader].pbcb.cb ); if ( pReaders[pFS->iReader].pbcb.max > pReaders[pFS->iReader].pFS->pNode->size - pReaders[pFS->iReader].offset ) pReaders[pFS->iReader].pbcb.max = pReaders[pFS->iReader].pFS->pNode->size - pReaders[pFS->iReader].offset; debug( true, CACHE_HIT_DETAIL_MSG, pFS->iReader, pFS->iFS, l, req.pbcb.max, req.offset ); pReaders[pFS->iReader].stats.bytesCached += l; } /*_________________________________________________________________________________*/ /* Part 4: if we have a 100% hit, respond directly & do a 'read-ahead' if needed. */ if ( req.pbcb.cb == req.pbcb.max ) { if ( pReaders[pFS->iReader].readType== rSLOW && pReaders[pFS->iReader].pbcb.cb == 0 && pReaders[pFS->iReader].pFirstReq == NULL && (pReaders[pFS->iReader].command & COMMAND_FILL_BUFFER) == 0 ) { debug( true, READ_AHEAD_REQUEST_MSG, pFS->iReader, pFS->iFS ); pReaders[pFS->iReader].command |= COMMAND_FILL_BUFFER; pReaders[pFS->iReader].pbcb.max= (RA_BUFFER_SIZE <= pReaders[pFS->iReader].pFS->pNode->size - (req.offset + req.pbcb.max) ) ? RA_BUFFER_SIZE : pReaders[pFS->iReader].pFS->pNode->size - (req.offset + req.pbcb.max); sem_post_or_die(pReaders[pFS->iReader].pSemPutReq ); } pthread_mutex_unlock_or_die( &pReaders[pFS->iReader].lThis ); return req.pbcb.cb; } else { /*_________________________________________________________________________________*/ /* Part 5: if the incoming read is not 'full', we queue our 'Request' to the reader*/ request_t **pPrev; req.pNextReq= NULL; for ( pPrev= &pReaders[pFS->iReader].pFirstReq; *pPrev != NULL; pPrev= &((*pPrev)->pNextReq) ) if ( req.pFS == (*pPrev)->pFS && req.offset + req.pbcb.max == (*pPrev)->offset ) { debug( true, REVERSED_SEQUENCE_MSG, pFS->iReader, pFS->iFS, req.offset, (*pPrev)->offset ); req.pNextReq= *pPrev; break; } *pPrev= &req; req.pSemResp= &semResp; sem_post_or_die(pReaders[pFS->iReader].pSemPutReq ); } } pthread_mutex_unlock_or_die( &pReaders[pFS->iReader].lThis ); /*_________________________________________________________________________________*/ /* Part 6: wait for the job to be done for us, free resources and reply to fuse. */ sem_wait_or_die( &semResp ); if ( req.error ) return -EIO; else return req.sizeRead; } /*====================================================================================\ | Page: 42 | | |==========/ | | Function: write_move() | |========== | | | | Description: | |============= | | Curl callback for the asynchronous readers. | | | | This function is sort of the heart of the asynchronous read. | | It is split in 2 distinct parts: | | -1) We are currently reading for a buffer given by the fuse callback. | | -2) ... we are not... which means we read on our internal read-ahead buffer. | | | | About mutexes optimisation: | | the reader structure is accessed both by the fuse wrapper (previous function) and | | by the asynchronous reader (including this function that runs on the same thread | | although it is 'called-back' by curl). So, to avoid, as much as possible, having | | to lock on the most frequent situation which is: we read bytes for the fuse | | callback, when a request comes from the wrapper, we take it out of the queue | | (under locks) and put it on the readers 'pReq' which is ONLY accessed by this | | thread. That is why the begining of this function (most cases excepted 'Slow' mode | | ... but as it is 'Slow', it's not a problem doing some CPU to lock!) can be done | | without any lock. | | Some fields are set only by one function/thread, namely the statistics. | | For example, the best & only place we count incoming bytes is at this callback, | | and same for read-aborts. Read count, on the other hand, is updated only by | | the fuse threads. So for these fields, there is obviously no need locking. | | | | -1) a) Simply puts incoming data in the fuse buffer. | | b) If there are more byte than the buffer can hold, it means either we are in | | 'Sequential' read, or we have 'Globed' 2 consecutive sequential read in a | | single bigger read. Anyway, now we must determine the best use of the bytes | | we have in excess. | | Now we have to lock, as we need to access parts also read by the fuse threads| | c) We look if we have a new request in the queue. If so, we check if it is the | | continuation of the previous request. Most of the time, it should be. But if | | we guessed wrong with SEQ, or unlikely if we have to loop on very short | | request, we might find something not 'sequential'. | | So, if it IS the contination of the previous read, this request will become | | our new pReq (current read to fuse). We then decrement the semaphore, as we | | are handling this new request from here, we don't want the main even loop to | | try to fulfill a request we are removing from the queue! (which will most | | likely GP Fault!). We do also set the offsets and index accordingly. | | d) And so, if there is indeed a continuation, we do the same as 1 a): store the | | bytes we have left on this new request. Note that at this point, as we used | | our fill_buffer utility, if this next request is already full (could happen | | although very unlikely, if we get short request ex. 4096, and had a buffer | | of 16384 from which we took let's say 8192 on the 1a) we have 'responded' | | Thus we might still have bytes in excess, and that's why we use a 'while' | | e) And else, it is the situation where we don't have a continuation (either we) | | have no next request at all, or we had one but not the continuation. | | I) Avoid useless buffering on a closed file (can happens for example if the | | user 'breaks' a copy) or if we get a non-continuous request on the same | | file. We can have another file here, if the reader was in SEQ mode but was | | selected as 'best' for a new file and we got a 1st read on that new file. | | In this situation, we still keep the data from the original file as the | | awaited continuous request might come next. | | II) Last branch of this 1st part, we had excess bytes but no continuation | | request to store that, so we store the bytes in our internal 'buffer', | | hoping that we will get, at some point in time, the continuation request | | we needed. Note that we still have the 'while' loop in this situation | | although it is normally useless (but just in the paranoid case where curl | | had a huge read that overflows our buffer... we can deal with it!) | | | | -2) Here we had previously hit the 1-e-II): we started buffering, and data continue | | to arrive at our callback. The algorithm is similar to what we do on the first | | part, with minor variations. | | a) Stop buffering when we detect that the file has been closed. | | b) when no new incoming request, we continue buffering, and we log when too many | | bytes or buffer exactly full (happens in Slow mode). If we have byte in excess| | at this point we have to stop the reading because there is no other place to | | store next reads. | | c) and conversly, the last case is that we do indeed have a new incoming request | | I) if it matches our buffer, if so, we do like in 1-c) = this incoming request | | becomes our new current request, and thus we put our bytes on its buffer. | | Note that if the internal read-ahead was not empty, bytes would have been | | moved from it to the incoming request buffer by the fuse callback. Thus | | here, the comparison of offsets takes that into account. | | There is another slight variation from 1-c), we could be in Slow mode, | | doing a read-ahead of 128K (our internal buffer size). But the application | | can, at that moment, let's say, send 64K + 128K all sequential. If we were | | close to the end of the read-ahead, the 64K would be served directly by | | the fuse callback, and we get the next 128K sequential that goes beyond the | | point where we will stop reading (because read-ahead requests are sized to | | our internal buffer). So the difference, in this situation, is that we | | MUST NOT decrement the semaphore. On the contrary, we continue reading and | | when curl stops sending us bytes we end up in async_reader that will detect | | this situation and make another request. We do also move here to SEQ read | | as this situation happened because we were in Slow and now things seem to | | come faster... | | II) ... or it doesn't match and we stop reading. We have to set the current | | request to NULL, although we started in a branch where it was NULL, but | | 2.c-I) can set this to not NULL, and if the request was small we could have | | looped here with a not NULL value. | | ... and so, same as in 1), we loop to take into account small request that | | could result in several times where we have excess bytes to store. | | | \====================================================================================*/ size_t write_move( buffer, size, nmemb, pReader) void *buffer; size_t size; size_t nmemb; reader_t *pReader; { size_t l= size * nmemb; size_t n, m; request_t *pReq; if ( l == 0 ) return 0; pReader->stats.bytesRead += l; if ( pReader->pReq != NULL ) { /*_________________________________________________________________________________*/ /* Part 1: we are reading something for the fuse callback's buffer. */ /* 1-a) most of the time (SEQ and RAND) we are simply here, we fill byte in the */ /* buffer given by fuse and sent to us via */ /* the request. */ n= fill_request( pReader, buffer, l ); if ( l > n ) { /* 1-b) here we have byte in excess and must determine the best way to */ /* use them. Happens (mostly) on SEQ reads */ m= l- n; pthread_mutex_lock_or_die( &pReader->lThis ); { do { pReq= pReader->pFirstReq; if (pReq != NULL) { /* 1-c) in this situation we have a new request queued. As we */ /* are normally only here for SEQ or globed */ /* RAND, the request is probably the continu-*/ /* ation of the previous one... but as we */ /* do a 'while' we must anyway check that... */ /* Also, we could have guessed wrong with SEQ*/ if ( pReq->offset== pReader->lastOffset && pReq->pFS== pReader->pLastFS ) { debug( true, SEQ_READ_CONTINUATION_MSG, pReq->pFS->iReader, pReq->pFS->iFS, pReq->pbcb.max, pReq->offset ); pReader->pFirstReq = pReq->pNextReq; pReader->pLastFS = pReq->pFS; pReader->lastOffset= pReq->offset + pReq->pbcb.max; sem_wait_or_die( pReader->pSemPutReq ); } else pReq= NULL; /* If not continuation just set pReq to NULL */ } /* And now our current request is what we had*/ pReader->pReq= pReq; /* on the local pReq: the next request if it */ /* is the continuation, or NULL if there were*/ /* no next request or if it was no the cont- */ /* inuation */ if (pReq != NULL) { /* 1-d) same as the 1-a) but with our 'new' request we got */ m= fill_request( pReader, ((char *)buffer) + n, m ); n+= m; } else /* 1-e) Here we have no continuation request. */ if ( pReader->nFiles == 0 || ( pReader->pFirstReq != NULL && pReader->pFirstReq->pFS== pReader->pLastFS ) ) { /* 1-e-I) avoid buffering if file is closed or we have a */ /* non-continuous on the same file. */ debug( true, SEQ_READ_STOPPED_MSG, (unsigned int)(pReader - pReaders) ); l= 0; pReader->stats.nAborts++; } else { /* 1-e-II) and lastly we start buffering bytes. */ debug( true, START_BUFFERING_MSG, pReader->pLastFS->iReader, pReader->pLastFS->iFS ); if ( pReader->pFS != NULL && pReader->pbcb.cb != 0 ) { debug( true, BUFFER_BYTES_LOST_MSG, pReader->pFS->iReader, pReader->pFS->iFS, pReader->pbcb.cb, pReader->pFS->pNode->pName, pReader->offset ); pReader->stats.bytesLost += l; } pReader->offset= pReader->lastOffset; pReader->pFS= pReader->pLastFS; pReader->pbcb.cb= 0; pReader->pbcb.max= (RA_BUFFER_SIZE <= pReader->pFS->pNode->size - pReader->offset ) ? RA_BUFFER_SIZE : pReader->pFS->pNode->size - pReader->offset; m= store_data( &pReader->pbcb, ((char *)buffer) + n, m ); l= m + n; } } while ( pReq != NULL && n < l ); } pthread_mutex_unlock_or_die( &pReader->lThis ); } } else { /*_________________________________________________________________________________*/ /* Part 2: we have started buffering and we receive now more byte to handle. */ pthread_mutex_lock_or_die( &pReader->lThis ); { if ( pReader->pFS == NULL ) { /* 2-a) the file have been released, thus we stop buffering. */ debug( true, STOP_BUFFERING_ON_FILE_CLOSED_MSG, (unsigned int)(pReader - pReaders) ); l= 0; pReader->stats.nAborts++; } else { n= 0; do { pReq= pReader->pFirstReq; if (pReq == NULL) { /* 2-b) when no new incoming request, we continue buffering. */ /* And log when too many bytes or buf. full */ m= store_data( &pReader->pbcb, buffer + n, l - n); if (l != m + n) { debug( true, STOP_BUFFERING_ON_BUFFER_FULL_MSG, pReader->pFS->iReader, pReader->pFS->iFS ); l= m + n; pReader->readType= rSLOW; pReader->stats.nAborts++; } else debug( pReader->pbcb.cb == pReader->pbcb.max, BUFFER_FULL_MSG, pReader->pFS->iReader, pReader->pFS->iFS ); } else /* 2-c) and conversly, here we have a new request incoming. */ if ( pReq->offset + pReq->pbcb.cb == pReader->offset && pReq->pFS == pReader->pFS ) { /* 2-c-I) Either it matches our read coordinates. */ /* Then it becomes our new current request */ debug( true, CONTINUATION_AFTER_BUFFERING_MSG, pReq->pFS->iReader, pReq->pFS->iFS, pReq->pbcb.max, pReq->offset ); pReader->pReq = pReq; pReader->pLastFS = pReq->pFS; pReader->lastOffset= pReq->offset + pReq->pbcb.max; if ( pReader->lastOffset <= pReader->endRead ) { pReader->pFirstReq = pReq->pNextReq; sem_wait_or_die( pReader->pSemPutReq ); } else pReader->readType= rSEQ; m= fill_request( pReader, ((char *)buffer) + n, l - n); n+= m; } else { /* 2-c-II) ... or it doesn't match and we stop reading. */ debug( true, STOP_BUFFERING_ON_NON_SEQ_READ_MSG, pReq->pFS->iReader, pReq->pFS->iFS, pReq->offset, pReader->offset, pReader->pFS->iFS ); l= 0; pReader->stats.nAborts++; pReader->pReq = NULL; pReader->readType= rRAND; } } while ( pReq != NULL && n < l ); } } pthread_mutex_unlock_or_die( &pReader->lThis ); } return l; } /*====================================================================================\ | Page: 43 | | |==========/ | | Function: compute_path() | |========== | | | | Description: | |============= | | This function computes the absolute path (from the Freebox root) given a FSNode. | | | | It is used when doing a 'read-ahead' because at this point we don't have the whole | | path but only a pointer on the FSNode corresponding to our file. This allows us | | to compute the absolute path (by recursing up to the root of the structure). | | | \====================================================================================*/ char * compute_path( pNode ) FSNode *pNode; { char *pPath; size_t l; if (pNode == &FSRoot) { pPath = malloc(1); if (pPath != NULL) *pPath= '\0'; } else { pPath= compute_path( pNode->parent ); if (pPath != NULL) { l= strlen(pPath); pPath= realloc( pPath, l + strlen( pNode->pName ) + 2 ); if (pPath != NULL) { *(pPath + l)= '/'; strcpy( pPath + l + 1, pNode->pName ); } } } return pPath; } /*====================================================================================\ | Page: 44 | | |==========/ | | Function: fbx_read() | |========== | | | | Description: | |============= | | This function does the actual read on the server (Freebox Revolution/V6 ). | | | | It is in fact part of async_read that is the only one calling that. But as it is | | the only part of the read that is 'server-dependant', better to isolate it here. | | | | It does the following steps: | | - computes the boundaries of the read request to send to the server. They depend on | | whether it is a 'read-ahead' request or not, the read mode: SEQ or not, and even | | in cases where RAND is forces (--no-cache) we optimise if we found 2 consecutive | | sequential read, and do a single one instead. | | - Then we prepare the header (Range) from that, and put the post option. | | - Next we do the request. | | - If we get a HTTP status 200 or 302, it means our cookie expires (or was cancelled)| | if so, we attempt a reconnexion, and do the read again. To avoid trying that | | endlessly, the loop limit is 2 (meaning only 1 reconnexion attempt) | | It is safe to do the reconnexion here and not is write-move, because the first | | thing the server sends back are headers. Thus if we don't have the expected 206 | | we won't even go to write_move, that will be caught immediately by our header | | callback which would stop curl and return. It is also why we don't need to test | | | | Note on locking: this function do not require locking. 'Command' is also written | | by fuse, but is passed as parameter (and was read under lock). pbcb.max is also | | written in fuse callback, but in this case, as we were called on the command | | it means the internal buffer is empty plus other conditions (see fbx_read_wrapper) | | that implies it will not be written at the same time when we need to read it here. | | So we don't need locking at all in this function. | | | \====================================================================================*/ static int fbx_read( iReader, command, ppPost ) int iReader; int command; char **ppPost; { char aRange[64]; char *pPath; CURLcode res; int nRetry; off_t startRange, endRange; reader_t *pReader= pReaders + iReader; /*_________________________________________________________________________________*/ /* Part 1: compute the boudaries of the actual read request. */ if ( command & COMMAND_FILL_BUFFER) /* If we have a 'read-ahead' command, the */ { /* The start is the offset we have for our */ startRange= pReader->offset; /* internal buffer, and max was set when the */ /* command was sent */ endRange= startRange + pReader->pbcb.max; debug( true, READ_AHEAD_MSG, iReader, pReader->pFS->iFS, pReader->pbcb.max, pReader->offset ); /* We have to compute the path from the Node */ /* because we don't keep path from the fuse */ /* request (would have to alloc/free mem & */ /* we have all the info in the Freebox Tree) */ pPath= compute_path( pReader->pFS->pNode ); if ( pPath == NULL ) return -1; *ppPost= URIescape( pPath, pPostReadRoot, sPostReadRoot ); free( pPath ); } else { /* Start is offset given by fuse + cb */ /* because some bytes could have been moved */ /* from the internal rea-ahead buffer. */ startRange= pReader->pReq->offset + pReader->pReq->pbcb.cb; /* End is: end of file for sequential mode. */ if (pReader->readType == rSEQ) endRange= pReader->pReq->pFS->pNode->size; else { /* Special optimisation in random mode when */ /* we get 2 requests that are sequential, in */ /* this case, the end spans the 2 requests. */ if ( pReader->pReq->pNextReq != NULL && pReader->pReq->pNextReq->offset== pReader->pReq->offset + pReader->pReq->pbcb.max ) { endRange= pReader->pReq->pNextReq->offset + pReader->pReq->pNextReq->pbcb.max; debug( true, GLOBING_2_REQUESTS_MSG, iReader, pReader->pReq->pFS->iFS, pReader->pReq->pbcb.max, pReader->pReq->pNextReq->pbcb.max, pReader->pReq->offset ); } else /* In 'normal' RAND, the end is what fuse */ /* gave to us (and we copied in the request)*/ endRange= pReader->pReq->offset + pReader->pReq->pbcb.max; } *ppPost= URIescape( pReader->pReq->pPath, pPostReadRoot, sPostReadRoot ); } if ( *ppPost == NULL ) return -1; /*_________________________________________________________________________________*/ /* Part 2: misceallanous and curl options (headers, post) preparation */ pReader->endRead= endRange; pReader->stats.nReads++; snprintf( aRange, sizeof( aRange ), FORMAT_RANGE_HEADER, startRange, endRange - 1 ); for ( nRetry=0; nRetry< 2; nRetry ++ ) { struct curl_slist *pHeaderList; pHeaderList = curl_slist_append( NULL, aRange); if ( pHeaderList == NULL ) return -1; curl_easy_setopt_or_die( iReader, CURLOPT_HTTPHEADER, pHeaderList); curl_easy_setopt_or_die( iReader, CURLOPT_POSTFIELDS, *ppPost ); pReader->statusCode= 0; res= curl_easy_perform( pReader->curl ); curl_slist_free_all( pHeaderList ); /*_________________________________________________________________________________*/ /* Part 3: if request went wrong we try a reconnexion and 2nd request. */ if ( pReader->statusCode== HTTPE_FOUND || pReader->statusCode== HTTPE_OK ) { lprintf ( LOG_NOTICE, true, INVALID_COOKIE_MSG, pReader->statusCode ); curl_easy_cleanup( pReader->curl ); pReader->curl= NULL; init_reader( iReader, true ); lprintf ( LOG_NOTICE, true, RECONNEXION_SUCCESS_MSG ); } else break; } if ( res != CURLE_OK && ! (pReaders[iReader].pReq==NULL && res== CURLE_WRITE_ERROR) ) return -1; else return 0; } /*====================================================================================\ | Page: 45 | | |==========/ | | Function: init_reader() | |========== | | | | Description: | |============= | | Initialises a reader structure. | | | | Param.: the index of the reader to initialize (0 to maxReaders). | | a flag for complete/partial initialisation. | | | | This function does NOT allocates the reader structure itself. This is because it | | is supposed to be allocated contiguously so that we can use either the index or | | a pointer. The allocation must be done prior to call this function. | | | | The FIRST time it is called the function MUST be called with the 'false' flag that | | will do a partial initialisation so that we know what is to be done for subsequent | | call. | | | | The next times, the function SHOULD be called with the 'true' flag. Then the | | necessary initialisations will be done. | | - allocation of the semaphore and the internal buffer. | | - initialisation of the semaphore. | | - initialisation of the curl handle. | | - initialisation of commonly used curl options. | | - connexion to the freebox. | | - preparation of curl options for reads. | | | | The 'two call modes' are also useful to allocate readers 'only when necessary'. | | In the main, we allocate the structure for readers but do only a minimal init with | | the false flag. This ensures for example that we don't allocate the 128K buffer | | when not needed. Then during the read process, if we need more readers we do the | | complete initialisation and proceed with the start of the new thread. Thus the | | user could specify 10 readers, and end up using only 1 or 2!.. | | | \====================================================================================*/ void init_reader( iReader, fInitAll ) int iReader; bool fInitAll; { if ( fInitAll ) { /*_________________________________________________________________________________*/ /* Complete initialisation here. Divided in 2 parts, according to what needs init. */ if ( pReaders[iReader].pSemPutReq == NULL ) { /*_________________________________________________________________________*/ /* Part 1: if needed: memory and semaphore initialisations. */ if (fNoCache) /* When no-cache option is up, we don't want */ { /* the buffer, only the semaphore is required*/ /* The only pattern will be 'random' then. */ pReaders[iReader].pSemPutReq= malloc_or_die( sizeof(sem_t) ); pReaders[iReader].pbcb.pb= NULL; pReaders[iReader].readType= rRAND; } else { pReaders[iReader].pSemPutReq= malloc_or_die( sizeof(sem_t) + RA_BUFFER_SIZE ); pReaders[iReader].pbcb.pb= (void *)(pReaders[iReader].pSemPutReq + 1); pReaders[iReader].readType= rINIT; } sem_init_or_die( pReaders[iReader].pSemPutReq, 0, 0 ); pReaders[iReader].pbcb.cb= 0; pReaders[iReader].pbcb.max= RA_BUFFER_SIZE; pReaders[iReader].pReq= NULL; pReaders[iReader].command= 0; pReaders[iReader].pLastFS= NULL; pReaders[iReader].lastOffset= 0; pReaders[iReader].pFS= NULL; pReaders[iReader].offset= 0; pReaders[iReader].pFirstReq= NULL; lprintf( LOG_INFO, true, READER_INITIALIZED_MSG, iReader ); } if ( pReaders[iReader].curl == NULL ) { /*_________________________________________________________________________*/ /* Part 2: if needed: we initialize curl and do the actual connexion */ pReaders[iReader].curl= curl_easy_init(); if ( pReaders[iReader].curl == NULL ) { lprintf( LOG_CRIT, true, CURL_INIT_FAILED_MSG); exit(EXIT_FAILURE); } /* curl_easy_setopt(FSState.curl,CURLOPT_VERBOSE,1L); // Uncomment to debug*/ /* Some curl elements we will use all along */ /* our curl calls. We initialise then once */ /* and for all here. */ /* Curl errorbuffer string */ curl_easy_setopt_or_die( iReader, CURLOPT_ERRORBUFFER, pReaders[iReader].curlErrBuf ); /* initialising the cookie engine. */ curl_easy_setopt_or_die( iReader, CURLOPT_COOKIEFILE, "" ); /* Parser for header (to get HTTP statuscode)*/ curl_easy_setopt_or_die( iReader, CURLOPT_HEADERFUNCTION, parse_status_code ); curl_easy_setopt_or_die( iReader, CURLOPT_HEADERDATA, &pReaders[iReader].statusCode ); /* Here we do the login to the server. */ server_login( iReader ); } /* And those are options used for the reads */ curl_easy_setopt_or_die( iReader, CURLOPT_HEADERFUNCTION, parse_status_range ); curl_easy_setopt_or_die( iReader, CURLOPT_URL, pFbxDownloadURI ); curl_easy_setopt_or_die( iReader, CURLOPT_WRITEFUNCTION, write_move ); /* Pbcb pointer for read to dump it's data */ curl_easy_setopt_or_die( iReader, CURLOPT_WRITEDATA, &pReaders[iReader] ); } else { /*_________________________________________________________________________________*/ /* Minimal initialisation here. Mainly zeroes + init of the mutex (we always need) */ pReaders[iReader].nFiles = 0 ; pReaders[iReader].stats.bytesRead = 0 ; pReaders[iReader].stats.bytesCached = 0 ; pReaders[iReader].stats.bytesLost = 0 ; pReaders[iReader].stats.nFS = 0 ; pReaders[iReader].stats.nReads = 0 ; pReaders[iReader].stats.nAborts = 0 ; pReaders[iReader].pSemPutReq = NULL; pReaders[iReader].curl = NULL; pthread_mutex_init( &pReaders[iReader].lThis, NULL); } } /*====================================================================================\ | Page: 46 | | |==========/ | | Function: start_reader() | |========== | | | | Description: | |============= | | This starts the reader process which index is given as parameter. | | | | IMPORTANT: this function must be called inclosed inside a section locked by the | | mutex of the reader we want to start. | | This reader must have been FULLY initialised because now that it will run on | | a thread, requests can be dispatched to it. | | | | Why do we need to do so: | | we start the async reader by passing to it the reader index variable that resides | | on our stack. Thus if this function exits before the reader thread had time to | | read and save the variable, we would end up with an un-determined variable in the | | new thread. To avoid that, we must be sure the thread got our variable. | | So the process is: | | - This thread (in this function): | | a) locks the mutex (to be done by the caller) | | b) launches the async-reader thread with the parameter. | | c) waits on the readers semaphore | | d) unlocks the mutex (to be done by the caller) | | - The async reader does | | a) save the variable passed. | | b) post on the readers semaphore | | c) locks the mutex | | d) unlocks the mutex | | | | One might believe that the semaphore could be enough. But as the async_reader main | | loop waits on that semaphore, if we don't use mutexes, there are chances that the | | post on the semaphore is taken by the wait on the same thread resulting with a | | GP Fault. Even with lower priority on the async_reader this still happens | | (occasionally). This happens more on multi-processor/multi-core environment, | | because although we set the semaphore free with a post and have less priority, | | the system might have a free processor to run our low priority new thread and then| | the 'bug' happens. To be sure it does NEVER happen, we lock. What occurs then is: | | - This thread will wait in c) for the async to post (lock or not it waits anyway!) | | - The async-thread starts, reads the variables and posts, but can't go past to c) | | because the mutex is locked by us. So here, we are sure the post will not go to | | the event loop on the launched thread (GP Fault + us kept waiting!) but will | | come back to us. And we are then sure the new thread got our variable! | | - Then once we get the semaphore back, we unlock and the launched thread is | | unlocked too so it can continue safely to its main loop, waiting its semaphore. | | | \====================================================================================*/ void start_reader( iReader ) int iReader; { lprintf( LOG_CRIT, pthread_create( pThreadRead + iReader, NULL, async_reader, &iReader ), THREAD_CREATION_FAILED_ERR_MSG ); sem_wait_or_die( pReaders[iReader].pSemPutReq ); } /*====================================================================================\ | Page: 47 | | |==========/ | | Function: async_reader() | |========== | | | | Description: | |============= | | This is the main event loop for our asynchronous reading process. | | | | It does: | | - initialisation of the thread. Mainly, we read our parameter (passed during the | | call of pthread_create). That is because obviously, the exact same code runs | | for each reader. The only difference is the index of the reader. To be sure to | | get that index, we have to do careful locking that is explained in start_reader. | | It does also lower the priority of the thread. | | ABOUT PRIORITIES: | | the main thread (our main) runs at a priority lowered by 1 (from the lauching | | shell) and the async_reads run at a priority lower by 2. This is meant to give | | CPU time to the calling application to give us as many simultaneous requests at | | possible. In order to achieve that, we want to have less priority than those | | applications so that we don't get scheduled when they are still sending reads. | | That is because the more simultaneous requests we get, the more accurate are our | | 'guesses' on the read mode. Some optimisations such as globing 2 consecutive | | random requests, even need to have 2 requests present. Same goes for priority | | between fuse and the readers. We want fuse to run with more priority as the main | | part that determines mode is this function, thus the fuse callback need to run | | at higher priority to process all the reads and send us as many simultaneous | | requests as possible. The fuse callback does also some reordering on the request | | queue. Obviously, if it was preempted by this thread, it would be less efficient | | as the queue would have more chances to be empty. | | But do also note, as it is explained in start_reader, that this priority setting | | do not prevent our program to run in parallel with the user application even at | | when we have lower priority. That happens when the user application is a mono | | threaded application and we are running in SMP ('multi-processor'). And same goes | | between fuse callback and these threads. Even at different priority, this thread | | can still read the queue while the fuse callback puts element in it, all that in | | parallel on a multiprocessor. | | | | After the initialisation part, we have the main event loop. It does: | | - 'wait' for something to do (on the semaphore, when fuse callback send us requests | | or commands, they are first stored on the structure, then the semaphore is incre- | | mented, so that the event loop stops waiting and gets the stuff to process). | | - determine ('guess') the reading mode. | | There is a special mode 'init'. From this mode, either we move to 'calc' or if | | we are at offset 0, and we already have 2 requests (see priority discussion) and | | they are sequential, we move directly to sequential. | | In other cases, we move to sequential or random according to the current request | | and the end offset of the last read we processed on that files. | | - then we do the actual request using fbx_read (see previous page) | | - and finally we respond to the caller (in this case the fuse callback). We don't | | do this final part if pReq (current request) is NULL, because this means we | | already responded from write_move, and when write_move exited, we were reading | | on our internal request. We don't respond neither when the end of the last request| | goes beyond the end or current read. In that situation, write_move exited with | | and incomplete request, but didn't remove the semaphore nor the request from the | | queue. So we will then go back to our main event loop, get the semaphore and the | | same request to 'finish' filling it. This happens only in Slow mode. | | | \====================================================================================*/ void * async_reader( pUserData ) void *pUserData; { int iReader; reader_t *pReader; int command; char *pPost; int err; /*_________________________________________________________________________________*/ /* Part 1: intialisations: get the param, priority, syncrhonise */ iReader= *(int *)pUserData; pReader= &pReaders[iReader]; { pid_t tid; tid = syscall(SYS_gettid); setpriority(PRIO_PROCESS, tid, 2); } sem_post_or_die( pReaders[iReader].pSemPutReq ); pthread_mutex_lock_or_die( &pReader->lThis ); pthread_mutex_unlock_or_die( &pReader->lThis ); lprintf( LOG_INFO, true, READ_THREAD_STARTED_MSG, iReader ); do { /*_________________________________________________________________________________*/ /* Part 2: main event, that's our 'wait'! */ sem_wait_or_die( pReader->pSemPutReq ); pthread_mutex_lock_or_die( &pReader->lThis ); { command= pReader->command; if ( command & COMMAND_FILL_BUFFER ) { /* Special case when we have a 'fill-buffer' */ /* command from the fuse callback. */ pReader->command ^= COMMAND_FILL_BUFFER; if ( pReader->pFirstReq != NULL ) { /* As it is always more urgent to read what */ /* is really needed by the application than */ /* to do the read-ahead, if between the time */ /* the command was given and now that we */ /* handle it, we have received another read */ /* we just ignore the command and do the read*/ debug( true, READ_AHEAD_IGNORED_ON_OTHER_REQUESTS_MSG, iReader ); command= 0; sem_wait_or_die( pReader->pSemPutReq ); } } if ( command== 0 ) { /* When no command (or ignored) we have a */ /* read to handle, then we determine the */ /* read mode (see comments in header) */ pReader->pReq= pReader->pFirstReq; pReader->pFirstReq= pReader->pReq->pNextReq; if (pReader->nFiles > 1 || fNoCache ) pReader->readType= rRAND; else if ( pReader->readType == rINIT ) if ( pReader->pReq->offset == 0 && pReader->pReq->pNextReq != NULL && pReader->pReq->pNextReq->offset== pReader->pReq->offset + pReader->pReq->pbcb.max ) { pReader->readType= rSEQ; debug( true, IMMEDIATE_SEQ_READ_MSG, iReader, pReader->pReq->pFS->iFS ); } else pReader->readType= rCALC; else if ( pReader->pReq->offset == pReader->lastOffset || ( pReader->pReq->pNextReq != NULL && pReader->pReq->pNextReq->offset== pReader->pReq->offset + pReader->pReq->pbcb.max ) ) { if (pReader->readType != rSEQ) { debug( true, SEQUENTIAL_READ_MSG, iReader, pReader->pReq->pFS->iFS ); pReader->readType= rSEQ; } } else if (pReader->readType == rSLOW && pReader->pReq->offset == pReader->offset ) { debug( true, SEQUENTIAL_READ_FROM_SLOW_MSG, iReader, pReader->pReq->pFS->iFS ); pReader->readType= rSEQ; } else if (pReader->readType != rRAND) { debug( true, RANDOM_READ_MSG, iReader, pReader->pReq->pFS->iFS ); pReader->readType= rRAND; } pReader->pLastFS= pReader->pReq->pFS; pReader->lastOffset= pReader->pReq->offset + pReader->pReq->pbcb.max; debug( true, INCOMING_REQUESTS_MSG, iReader, pReader->pReq->pFS->iFS, pReader->pReq->pbcb.max, pReader->pReq->offset ); } } pthread_mutex_unlock_or_die( &pReader->lThis ); /* When we receive this command we just need */ /* to exit the function so that the thread */ /* terminates and can be 'joined'. It is */ /* sent by the destroy handle, for instance */ /* when fusermount -u is invoked by the user.*/ if ( command & COMMAND_STOP ) return NULL; /*_________________________________________________________________________________*/ /* Part 3: do the actual request. */ pPost= NULL; err= fbx_read( iReader, command, &pPost ); /*_________________________________________________________________________________*/ /* Part 4: respond to the fuse callback (when necessary) */ if (pReader->pReq != NULL || pReader->lastOffset > pReader->endRead) { if (err) pReader->pReq->sizeRead= 0; else pReader->pReq->sizeRead= pReader->pReq->pbcb.max; pReader->pReq->error= err; Respond( pReader->pReq ); pReader->pReq= NULL; } free (pPost); } while(1); } /*====================================================================================\ | Section | | |==========/ | | As usual, last section is our 'main' that orchestrates the overall operations. | | | \====================================================================================*/ /*====================================================================================\ | Page: 50 | | |==========/ | | Function: fuse_operations | |========== | | | | Description: | |============= | | This is the fuse_operations structure where we specify all the callbacks we have | | implemented for this filesystem. | | | \====================================================================================*/ static struct fuse_operations fbx_oper = { .getattr = fbx_getattr_wrapper, .readdir = fbx_readdir_wrapper, .open = fbx_open_wrapper, .release = fbx_release_wrapper, .read = fbx_read_wrapper, .destroy = fbx_destroy_wrapper, .init = fbx_init, }; /*====================================================================================\ | Page: 51 | | |==========/ | | Function: main() | |========== | | | | Description: | |============= | | As a main principle, everything we can do in the initial phase we do it, so that | | during the fuse part things to do are as quick as possible. For instance if the | | root of the target mount (start directory on the Freebox) needs escaping we do | | it at init so that we don't need, during fuse reads, to scan the whole path but | | only the path relative to that root. | | So that is where we prepare those 'semi-constants', during the init phase. | | | | Part 1: is about parameters acquisition and preparing 'semi-constants' | | | | Part 2: is where we start curl and initialize the whole tree of dirs/files | | | | Part 3: simply mark the init as done and call fuse_main | | | | Part 4: when fuse returns (normally or error) we do the cleanup | | | \====================================================================================*/ int main (argc, argv) int argc; char **argv; { struct curl_slist *headerlist=NULL; char *p=NULL; char *pConfigFile=NULL; char *pMountpoint=NULL; int i, res; setpriority(PRIO_PROCESS, 0, 1); /*_________________________________________________________________________________*/ /* Part 1: is about parameters acquisition and preparing 'semi-constants' */ if ( scan_arguments( argc, argv, &p, &pConfigFile, &pMountpoint ) ) { /* We come here if scan_arguments returned */ /* non-zero. Zero is returned when we have */ /* options -h or -V, which will skip all the */ /* rest of the initialisation and proceeed */ /* to fuse_main to display help/version. */ lprintf( LOG_INFO, true, PARAM_READ_OK_MSG ); check_URI( p ); lprintf( LOG_INFO, true, FREEBOX_URI_MSG, p ); lprintf( LOG_INFO, true, FREEBOX_ROOT_MSG, pFbxRoot ); /* We prepare once and for all the URI (see */ /* main comment = optimisation) */ pFbxLoginURI= alloc_cat( p, "/login.php", NULL ); pFbxLogoutURI= alloc_cat( p, "/login.php?logout=1", NULL ); pFbxReadDirURI= alloc_cat( p, "/fs.cgi" , NULL ); pFbxDownloadURI= alloc_cat( p, "/get.php" , NULL ); check_mountpoint( pMountpoint ); lprintf( LOG_INFO, true, LOCAL_MOUNTPOINT_MSG, pMountpoint ); if (pFbxPasswordPost==NULL) { get_password( pConfigFile ); } p=pFbxPasswordPost; pFbxPasswordPost= alloc_cat( "login=freebox&passwd=", p, NULL ); free( p ); /*_________________________________________________________________________________*/ /* Part 2: is where we start curl and initialize the whole tree of dirs/files */ curl_global_init(CURL_GLOBAL_NOTHING); pReaders= malloc_or_die( maxReaders * sizeof(reader_t) ); for ( i = 0; i < maxReaders ; i++ ) init_reader( i, false ); init_reader( 0, true ); lprintf( LOG_NOTICE, true, FBX_READ_TREE_MSG ); /* Now we need this header for subsequent */ /* calls to the server */ headerlist = curl_slist_append(headerlist, pJSONHeader); curl_easy_setopt_or_die( 0, CURLOPT_HTTPHEADER, headerlist); { void *pBuf= pReaders[0].pbcb.pb;/* Save the allocated buffer as it is going */ /* to be overwritten by write_alloc */ int nFiles, nDirs, nLinks; nFiles= nDirs= nLinks= 0; curl_easy_setopt_or_die( 0, CURLOPT_URL, pFbxReadDirURI ); curl_easy_setopt_or_die( 0, CURLOPT_HEADERFUNCTION, parse_status_code ); curl_easy_setopt_or_die( 0, CURLOPT_WRITEFUNCTION, write_alloc); curl_easy_setopt_or_die( 0, CURLOPT_WRITEDATA, &pReaders[0].pbcb ); store_server_tree( pFbxRoot, &FSRoot, &nFiles, &nDirs, &nLinks ); if ( nFiles == 0 && nDirs == 0 ) { lprintf( LOG_WARNING, true, NOTHING_ACCESSIBLE_WARN_MSG ); exit(0); } lprintf( LOG_NOTICE, true, TREE_SUMMARY_TOTAL_MSG, nFiles, nDirs, nLinks); lprintf( LOG_NOTICE, true, FBX_READ_TREE_SUCCESS_MSG ); pReaders[0].pbcb.pb= pBuf; /* And we restore the buffer now. */ } /* For root, modification date is that of the*/ /* child which has the biggest 'modification'*/ /* (which means last modified) */ for (i=0; i< FSRoot.detail.child.nNodes; i++) if ( FSRoot.modification < FSRoot.detail.child.pFirstNode[i].modification ) FSRoot.modification= FSRoot.detail.child.pFirstNode[i].modification; /* And we don't forget to free that now! */ curl_slist_free_all(headerlist); /* Last 'semi-constants' (uid/gid) then we */ /* set the 2 curl elements we need for */ /* downloading files, because it's the only */ /* thing we will do from now on. */ uid= geteuid(); gid= getegid(); lprintf( LOG_NOTICE, true, STARTING_FUSE_MSG ); } /*_________________________________________________________________________________*/ /* Part 3: simply mark the init as done and call fuse_main */ fInitDone=1; fDaemon= !fForeground; res= fuse_main( argcFuse, argvFuse, &fbx_oper, NULL); /*_________________________________________________________________________________*/ /* Part 4: when fuse returns (normally or error) we do the cleanup and freeing */ curl_global_cleanup( ); free_server_tree( &FSRoot ); free( pFbxLoginURI ); free( pFbxLogoutURI ); free( pFbxReadDirURI ); free( pFbxDownloadURI ); free( pFbxPasswordPost ); if ( strcmp( pFbxRoot, "/" ) != 0 ) { free( pFbxRoot ); free( pPostReadRoot ); } for (i= 1; i