""" ModelPagination Designed and Coded by Cal Leeming Many thanks to Harry Roberts for giving us a heads up on how to do this properly! You may also notice the class is almost exactly the same as the django pagination, give or take :) http://docs.djangoproject.com/en/dev/topics/pagination/?from=olddocs So this means, in most cases, you can use this as a drop in replacement. Although, if you are looking at using this, you would probably not just "drop it in" lol. ---------------------------------------------------------------------------- This is a super optimized way of paginating datasets over 1 million records. It uses MAX() rather then COUNT(), because this is super faster. EXAMPLE: >>> _t = time.time(); x = Post.objects.aggregate(Max('id')); "Took %ss"%(time.time() - _t ) 'Took 0.00103402137756s' >>> _t = time.time(); x = Post.objects.aggregate(Count('id')); "Took %ss"%(time.time() - _t ) 'Took 0.92404794693s' >>> This does mean that if you go deleting things, then the IDs won't be accurate, so if you delete 50 rows, you're exact count() isn't going to match, but this is okay for pagination, because for SEO, we want items to stay on the original page they were scanned on. If you go deleting items, then the items shift backwards through the pages, so you end up with inconsistent SEO on archive pages. If this doesn't make sense, go figure it out for yourself, its 2am in the morning ffs ;p Now, the next thing we do, is use id seeking, rather then OFFSET, because again, this is a shitton faster: EXAMPLE: >>> _t = time.time(); x = map(lambda x: x, Post.objects.filter(id__gte=400000, id__lt=400500).all()); print "Took %ss"%(time.time() - _t) Took 0.0467309951782s >>> _t = time.time(); _res = map(lambda x: x, Post.objects.all()[400000:400500]); print "Took %ss"%(time.time() - _t) Took 1.05785298347s >>> By using this seeking method (which btw, can be implemented on anything, not just pagination) on a table with 5 million rows, we are saving 0.92s on row count, and 1.01s on item grabbing. This may not seem like much, but if you have 1024 concurrent users, this will make a huge difference. If you have any questions or problems, feel free to contact me on cal.leeming [at] simplicitymedialtd.co.uk """ from django.core.paginator import Paginator , InvalidPage , EmptyPage from django.db.models import Max , Count , Q , F class ModelPagination : model = None items_per_page = None count = None page_range = [] def __init__ ( self , model , items_per_page ): self . model = model self . items_per_page = items_per_page self . count = self . model . aggregate ( Max ( 'id' ))[ 'id__max' ] self . num_pages = divmod ( self . count , self . items_per_page )[ 0 ] + 1 for i in range ( self . num_pages ): self . page_range . append ( i + 1 ) def page ( self , page_number ): if page_number > self . num_pages : raise EmptyPage , "That page contains no results" if page_number <= 0 : raise EmptyPage , "That page number is less than 1" start = self . items_per_page * ( page_number - 1 ) end = self . items_per_page * page_number object_list = self . model . filter ( id__gte = start , id__lt = end ) return ModelPaginationPage ( object_list , page_number , self . count , start , end , self ) class ModelPaginationPage : object_list = None number = None count = None start = None end = None paginator = None def __unicode__ ( self ): return "<Page %s of %s >" % ( self . number , self . count ) def __init__ ( self , object_list , number , count , start , end , paginator ): self . number = number self . count = count self . object_list = object_list self . start = start self . end = end self . paginator = paginator def has_next ( self ): return False if self . number >= self . count else True def has_previous ( self ): return False if self . number <= 1 else True def has_other_pages ( self ): return True if self . has_next or self . has_previous else False def next_number ( self ): return self . number + 1 def previous_number ( self ): return self . number + 1 def start_index ( self ): return self . start def end_index ( self ): return self . end ############################################################################### # OUR EXAMPLE USAGE ############################################################################### def archive ( request , * args , ** kwargs ): _t = time . time () # 4chan if kwargs . get ( 'feed' ) == '4chan' : ret = Post . objects url = '/archive/4chan-page-' else : raise Exception , "Invalid feed specified" # calculate what page we are on page_num = int ( args [ 0 ]) if args and args [ 0 ] else 1 # create the pagination object _items_per_page = 1000 pagination = ModelPagination ( Post . objects , 1000 ) # extract the items from the page page = pagination . page ( page_num ) items = map ( lambda x : { 'id' : x . get ( 'id' ), 'username' : x . get ( 'username' ), 'title' : make_title ( x . get ( 'message' ), x . get ( 'image_filename' ), x . get ( 'username' )), 'url' : "/fcp/ %s - %s .html" % ( make_title ( x . get ( 'message' ), x . get ( 'image_filename' ), x . get ( 'username' )), x . get ( 'id' )), 'partial_message' : x . get ( 'message' )[: 256 ] if x . get ( 'message' ) else None , 'created' : x . get ( 'created' ), 'image_url' : x . get ( 'image_url' ) }, page . object_list . values ( 'id' , 'username' , 'message' , 'image_filename' , 'created' , 'image_url' )) context = RequestContext ( request , { 'url' : url , 'page_num' : page_num , 'loading_time' : time . time () - _t , 'page' : page , 'items' : items , 'pagination' : pagination }) return render_to_response ( 'lazylittlegirl/archive/results.html' , context_instance = context ) """ <!-- Here is some example usage in a template, again this is just a copy and paste out of one of our projects, and not intended as a unit test or w/e --> <div id="content"> <ol> {% for item in items %} <li class="li1"> <div class="box1"> <a href="{{item.url}}" alt="{{item.title}}" title="{{item.title}}" target="_blank">Post #{{item.id}}</a> - {{item.created}} by {{item.username}} </div> </li> {% endfor %} </ol> <br /> <hr /> <div id="pagenumbers"><b>Pages :</b> {% for xpage in pagination.page_range %} {% if page.number == xpage %} [<b>{{xpage}}</b>] {% else %} <a title="Page {{xpage}} of {{pagination.num_pages}}" alt="Page {{xpage}} of {{pagination.num_pages}}" href="{{url}}{{xpage}}.html">{{xpage}}</a> {% endif %} {% endfor %} </div> """