Merge pull request #14 from SurelyYouareJoking/SurelyYouareJoking-patch-1

修改文件下载文件不完整问题

README.md

@@ -1,6 +1,6 @@
 # Fdfs_client py
 
-The Python interface to the Fastdfs Ver 4.06.
+The Python interface to the Fastdfs Ver 4.06.1.
 
 ## Installation
 
@@ -25,7 +25,8 @@ Class Fdfs_client:
 
 member functions:
 
-* upload_by_filename(self, filename, meta_dict = None)
+```
+upload_by_filename(self, filename, meta_dict = None)
   '''
   Upload a file to Storage server.
   arguments:
@@ -45,7 +46,7 @@ member functions:
             'Storage IP'      : storage_ip
         } if success else None
 
-* upload_by_buffer(self, filebuffer, file_ext_name = None, meta_dict = None)
+upload_by_buffer(self, filebuffer, file_ext_name = None, meta_dict = None)
   '''
   Upload a buffer to Storage server.
   arguments:
@@ -67,7 +68,7 @@ member functions:
         }
   '''
 
-* upload_slave_by_filename(self, filename, remote_file_id, prefix_name, \
+upload_slave_by_filename(self, filename, remote_file_id, prefix_name, \
                                  meta_dict = None)
   '''
   Upload slave file to Storage server.
@@ -90,7 +91,7 @@ member functions:
        }
   '''
 
-* upload_slave_by_buffer(self, filebuffer, remote_file_id, \
+upload_slave_by_buffer(self, filebuffer, remote_file_id, \
                                meta_dict = None, file_ext_name = None)
   '''
   Upload slave file by buffer
@@ -112,7 +113,7 @@ member functions:
        }
   '''
 
-* upload_appender_by_filename(self, local_filename, meta_dict = None)
+upload_appender_by_filename(self, local_filename, meta_dict = None)
   '''
   Upload an appender file by filename.
   arguments:
@@ -133,7 +134,7 @@ member functions:
 	   }
   '''
 
-* upload_appender_by_buffer(self, filebuffer, file_ext_name = None, meta_dict = None)
+upload_appender_by_buffer(self, filebuffer, file_ext_name = None, meta_dict = None)
   '''
   Upload a buffer to Storage server.
   arguments:
@@ -150,7 +151,7 @@ member functions:
        }
   '''
 
-* delete_file(self, remote_file_id)
+delete_file(self, remote_file_id)
   '''
   Delete a file from Storage server.
   arguments:
@@ -158,7 +159,7 @@ member functions:
        @return tuple ('Delete file successed.', remote_file_id, storage_ip)
   '''
 
-* download_to_file(self, local_filename, remote_file_id, offset = 0, down_bytes = 0)
+download_to_file(self, local_filename, remote_file_id, offset = 0, down_bytes = 0)
   '''
   Download a file from Storage server.
   arguments:
@@ -174,7 +175,7 @@ member functions:
        }
   '''
 
-* download_to_buffer(self, remote_file_id, offset = 0, down_bytes = 0)
+download_to_buffer(self, remote_file_id, offset = 0, down_bytes = 0)
   '''
   Download a file from Storage server and store in buffer.
   arguments:
@@ -189,7 +190,7 @@ member functions:
        }
   '''
 
-* list_one_group(self, group_name)
+list_one_group(self, group_name)
   '''
   List one group information.
   arguments:
@@ -197,7 +198,7 @@ member functions:
        @return Group_info,  instance
   '''
 
-* list_all_groups(self)
+list_all_groups(self)
   '''
   List all group information.
        @return dictionary {
@@ -206,7 +207,7 @@ member functions:
        }
   '''
 
-* list_servers(self, group_name, storage_ip = None)
+list_servers(self, group_name, storage_ip = None)
   '''
   List all storage servers information in a group
   arguments:
@@ -217,7 +218,7 @@ member functions:
        }
   '''
 
-* get_meta_data(self, remote_file_id)
+get_meta_data(self, remote_file_id)
   '''
   Get meta data of remote file.
   arguments:
@@ -225,7 +226,7 @@ member functions:
        @return dictionary, meta data
   '''
 
-* set_meta_data(self, remote_file_id, \
+set_meta_data(self, remote_file_id, \
                       meta_dict, op_flag = STORAGE_SET_METADATA_FLAG_OVERWRITE)
   '''
   Set meta data of remote file.
@@ -239,7 +240,7 @@ member functions:
        }
   '''
 
-* append_by_filename(self, local_filename, remote_fileid)
+append_by_filename(self, local_filename, remote_fileid)
   '''
   Append a file of Storage server
   arguments:
@@ -255,7 +256,7 @@ member functions:
        }
   '''
 
-* append_by_buffer(self, file_buffer, remote_fileid)
+append_by_buffer(self, file_buffer, remote_fileid)
   '''
   Append a file of Storage server
   arguments:
@@ -271,7 +272,7 @@ member functions:
        }
   '''
 
-* truncate_file(self, truncated_filesize, appender_fileid)
+truncate_file(self, truncated_filesize, appender_fileid)
   '''
   Truncate file in Storage server.
   arguments:
@@ -283,7 +284,7 @@ member functions:
        }
   '''
 	   
-* modify_by_filename(self, filename, appender_fileid, offset = 0)
+modify_by_filename(self, filename, appender_fileid, offset = 0)
   '''
   Modify a file in Storage server by filename.
   arguments:
@@ -296,7 +297,7 @@ member functions:
        }
   '''
 
-* modify_by_buffer(self, filebuffer, appender_fileid, offset = 0)
+modify_by_buffer(self, filebuffer, appender_fileid, offset = 0)
   '''
   Modify a file in Storage server by buffer.
   arguments:
@@ -308,7 +309,7 @@ member functions:
            'Storage IP' : storage_ip
        }
   '''
-
+```
 ### Connection Pools
 
 Behind the scenes, fdfs_client-py uses a connection pool to manage connections to
@@ -333,4 +334,3 @@ Special thanks to:
 
 * Andy Mccurdy, author of redis-py, referenced his code.
 * g.rodola, author sendfile module for python, g.rodola@gmail.com
-

fdfs_client/connection.py

@@ -23,7 +23,7 @@ class Connection(object):
     def __init__(self, **conn_kwargs):
         self.pid = os.getpid()
         self.host_tuple = conn_kwargs['host_tuple']
-        self.remote_port = None
+        self.remote_port = conn_kwargs['port']
         self.remote_addr = None
         self.timeout = conn_kwargs['timeout']
         self._sock = None
@@ -43,25 +43,17 @@ class Connection(object):
         except socket.error as e:
             raise ConnectionError(self._errormessage(e))
         self._sock = sock
-        # print '[+] Create a connection success.'
-        # print '\tLocal address is %s:%s.' % self._sock.getsockname()
-        # print '\tRemote address is %s:%s' % (self.remote_addr, self.remote_port)
-
-    def sendall(self, msg):
-        if not self._sock:
-            self.connect()
-        self._sock.sendall(msg)
-
-    def recv(self, len):
-        return self._sock.recv(len)
-
+        #print '[+] Create a connection success.'
+        #print '\tLocal address is %s:%s.' % self._sock.getsockname()
+        #print '\tRemote address is %s:%s' % (self.remote_addr, self.remote_port)
+        
     def _connect(self):
-        """Create TCP socket. The host is random one of host_tuple."""
-        self.remote_addr, self.remote_port = random.choice(self.host_tuple)
-        # print '[+] Connecting... remote: %s:%s' % (self.remote_addr, self.remote_port)
-        # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        # sock.settimeout(self.timeout)
-        sock = socket.create_connection((self.remote_addr, self.remote_port), self.timeout)
+        '''Create TCP socket. The host is random one of host_tuple.'''
+        self.remote_addr = random.choice(self.host_tuple)
+        #print '[+] Connecting... remote: %s:%s' % (self.remote_addr, self.remote_port)
+        #sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        #sock.settimeout(self.timeout)
+        sock = socket.create_connection((self.remote_addr, self.remote_port),self.timeout)
         return sock
 
     def disconnect(self):
@@ -70,8 +62,8 @@ class Connection(object):
             return
         try:
             self._sock.close()
-        except socket.error as e:
-            pass
+        except socket.error, e:
+            raise ConnectionError(self._errormessage(e))
         self._sock = None
 
     def get_sock(self):
@@ -109,7 +101,7 @@ class ConnectionPool(object):
     def _check_pid(self):
         if self.pid != os.getpid():
             self.destroy()
-            self.__init__(self.pool_name, self.conn_class, self.max_conn, **self.conn_kwargs)
+            self.__init__(self.conn_class, self.max_conn, **self.conn_kwargs)
 
     def make_conn(self):
         """Create a new connection."""
@@ -170,28 +162,33 @@ class ConnectionPool(object):
 
 # end ConnectionPool class
 
-def tcp_recv_response(conn, bytes_size, buffer_size=4096):
-    """Receive response from server.
+def tcp_recv_response(conn, bytes_size, buffer_size = 1024):
+    '''Receive response from server.
         It is not include tracker header.
         arguments:
         @conn: connection
         @bytes_size: int, will be received byte_stream size
         @buffer_size: int, receive buffer size
         @Return: tuple,(response, received_size)
-    """
-    recv_buff = []
+    '''
+    response = ''
     total_size = 0
+    total_bytes_size = bytes_size
     try:
-        while bytes_size > 0:
+        while 1:
+            if total_bytes_size - total_size <= buffer_size:
+                resp = conn._sock.recv(buffer_size)
+                response += resp
+                total_size += len(resp)
+                break
             resp = conn._sock.recv(buffer_size)
-            recv_buff.append(resp)
+            response += resp
             total_size += len(resp)
-            bytes_size -= len(resp)
-    except (socket.error, socket.timeout) as e:
-        raise ConnectionError('[-] Error: while reading from socket: (%s)' \
-                              % e.args)
-    return b''.join(recv_buff), total_size
-
+            
+    except (socket.error, socket.timeout), e:
+            raise ConnectionError('[-] Error: while reading from socket: (%s)' \
+                                    % e.args)
+    return (response, total_size)
 
 def tcp_send_data(conn, bytes_stream):
     """Send buffer to server.

fdfs_client/storage_client.py

@@ -89,16 +89,17 @@ def tcp_recv_file(conn, local_filename, file_size, buffer_size=1024):
     flush_size = 0
     remain_bytes = file_size
     with open(local_filename, 'wb+') as f:
-        while remain_bytes > 0:
+        diff_size = remain_bytes
+        while diff_size > buffer_size:
+            diff_size = remain_bytes - total_file_size
             try:
-                if remain_bytes >= buffer_size:
+                if diff_size >= buffer_size:
                     file_buffer, recv_size = tcp_recv_response(conn, buffer_size, \
                                                                buffer_size)
                 else:
-                    file_buffer, recv_size = tcp_recv_response(conn, remain_bytes, \
+                    file_buffer, recv_size = tcp_recv_response(conn, diff_size, \
                                                                buffer_size)
                 f.write(file_buffer)
-                remain_bytes -= recv_size
                 total_file_size += recv_size
                 flush_size += recv_size
                 if flush_size >= 4096: